HMVMXR0.cpp@ 78223

Last change on this file since 78223 was 78223, checked in by vboxsync, 6 years ago
VMM/HMVMXR0: Nested VMX: bugref:9180 Doxygen fixes.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 641.4 KB

Line
1	/* $Id: HMVMXR0.cpp 78223 2019-04-20 04:33:22Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27
28	#include <VBox/vmm/pdmapi.h>
29	#include <VBox/vmm/dbgf.h>
30	#include <VBox/vmm/iem.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#include <VBox/vmm/em.h>
35	#include <VBox/vmm/gim.h>
36	#include <VBox/vmm/apic.h>
37	#ifdef VBOX_WITH_REM
38	# include <VBox/vmm/rem.h>
39	#endif
40	#include "HMInternal.h"
41	#include <VBox/vmm/vm.h>
42	#include <VBox/vmm/hmvmxinline.h>
43	#include "HMVMXR0.h"
44	#include "dtrace/VBoxVMM.h"
45
46	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
47	#ifdef DEBUG_ramshankar
48	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
49	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
50	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
51	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
52	# define HMVMX_ALWAYS_TRAP_PF
53	# define HMVMX_ALWAYS_FLUSH_TLB
54	# define HMVMX_ALWAYS_SWAP_EFER
55	#endif
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	/** Use the function table. */
62	#define HMVMX_USE_FUNCTION_TABLE
63
64	/** Determine which tagged-TLB flush handler to use. */
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70	/** @name HMVMX_READ_XXX
71	* Flags to skip redundant reads of some common VMCS fields that are not part of
72	* the guest-CPU or VCPU state but are needed while handling VM-exits.
73	*/
74	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
75	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
76	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
77	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
78	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
79	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
80	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
81	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
82	/** @} */
83
84	/**
85	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
86	* guest using hardware-assisted VMX.
87	*
88	* This excludes state like GPRs (other than RSP) which are always are
89	* swapped and restored across the world-switch and also registers like EFER,
90	* MSR which cannot be modified by the guest without causing a VM-exit.
91	*/
92	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
93	\| CPUMCTX_EXTRN_RFLAGS \
94	\| CPUMCTX_EXTRN_RSP \
95	\| CPUMCTX_EXTRN_SREG_MASK \
96	\| CPUMCTX_EXTRN_TABLE_MASK \
97	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
98	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
99	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
100	\| CPUMCTX_EXTRN_TSC_AUX \
101	\| CPUMCTX_EXTRN_OTHER_MSRS \
102	\| CPUMCTX_EXTRN_CR0 \
103	\| CPUMCTX_EXTRN_CR3 \
104	\| CPUMCTX_EXTRN_CR4 \
105	\| CPUMCTX_EXTRN_DR7 \
106	\| CPUMCTX_EXTRN_HM_VMX_MASK)
107
108	/**
109	* Exception bitmap mask for real-mode guests (real-on-v86).
110	*
111	* We need to intercept all exceptions manually except:
112	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
113	* due to bugs in Intel CPUs.
114	* - \#PF need not be intercepted even in real-mode if we have nested paging
115	* support.
116	*/
117	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
118	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
119	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
120	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
121	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
122	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
123	\| RT_BIT(X86_XCPT_XF))
124
125	/** Maximum VM-instruction error number. */
126	#define HMVMX_INSTR_ERROR_MAX 28
127
128	/** Profiling macro. */
129	#ifdef HM_PROFILE_EXIT_DISPATCH
130	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
131	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
132	#else
133	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
134	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
135	#endif
136
137	/** Assert that preemption is disabled or covered by thread-context hooks. */
138	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
139	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
140
141	/** Assert that we haven't migrated CPUs when thread-context hooks are not
142	* used. */
143	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
144	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
145	("Illegal migration! Entered on CPU %u Current %u\n", \
146	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
147
148	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
149	* context. */
150	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
151	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
152	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
153
154	/** Helper macro for VM-exit handlers called unexpectedly. */
155	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
156	do { \
157	(a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
158	return VERR_VMX_UNEXPECTED_EXIT; \
159	} while (0)
160
161	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
162	/** Macro that does the necessary privilege checks and intercepted VM-exits for
163	* guests that attempted to execute a VMX instruction. */
164	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
165	do \
166	{ \
167	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
168	if (rcStrictTmp == VINF_SUCCESS) \
169	{ /* likely */ } \
170	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
171	{ \
172	Assert((a_pVCpu)->hm.s.Event.fPending); \
173	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
174	return VINF_SUCCESS; \
175	} \
176	else \
177	{ \
178	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
179	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
180	} \
181	} while (0)
182
183	/** Macro that decodes a memory operand for an instruction VM-exit. */
184	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
185	do \
186	{ \
187	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
188	(a_pGCPtrEffAddr)); \
189	if (rcStrictTmp == VINF_SUCCESS) \
190	{ /* likely */ } \
191	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
192	{ \
193	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
194	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
195	NOREF(uXcptTmp); \
196	return VINF_SUCCESS; \
197	} \
198	else \
199	{ \
200	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
201	return rcStrictTmp; \
202	} \
203	} while (0)
204
205	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
206
207
208	/*********************************************************************************************************************************
209	* Structures and Typedefs *
210	*********************************************************************************************************************************/
211	/**
212	* VMX transient state.
213	*
214	* A state structure for holding miscellaneous information across
215	* VMX non-root operation and restored after the transition.
216	*/
217	typedef struct VMXTRANSIENT
218	{
219	/** The host's rflags/eflags. */
220	RTCCUINTREG fEFlags;
221	#if HC_ARCH_BITS == 32
222	uint32_t u32Alignment0;
223	#endif
224	/** The guest's TPR value used for TPR shadowing. */
225	uint8_t u8GuestTpr;
226	/** Alignment. */
227	uint8_t abAlignment0[7];
228
229	/** The basic VM-exit reason. */
230	uint16_t uExitReason;
231	/** Alignment. */
232	uint16_t u16Alignment0;
233	/** The VM-exit interruption error code. */
234	uint32_t uExitIntErrorCode;
235	/** The VM-exit exit code qualification. */
236	uint64_t uExitQual;
237	/** The Guest-linear address. */
238	uint64_t uGuestLinearAddr;
239
240	/** The VM-exit interruption-information field. */
241	uint32_t uExitIntInfo;
242	/** The VM-exit instruction-length field. */
243	uint32_t cbInstr;
244	/** The VM-exit instruction-information field. */
245	VMXEXITINSTRINFO ExitInstrInfo;
246	/** Whether the VM-entry failed or not. */
247	bool fVMEntryFailed;
248	/** Whether we are currently executing a nested-guest. */
249	bool fIsNestedGuest;
250	/** Alignment. */
251	uint8_t abAlignment1[2];
252
253	/** The VM-entry interruption-information field. */
254	uint32_t uEntryIntInfo;
255	/** The VM-entry exception error code field. */
256	uint32_t uEntryXcptErrorCode;
257	/** The VM-entry instruction length field. */
258	uint32_t cbEntryInstr;
259
260	/** IDT-vectoring information field. */
261	uint32_t uIdtVectoringInfo;
262	/** IDT-vectoring error code. */
263	uint32_t uIdtVectoringErrorCode;
264
265	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
266	uint32_t fVmcsFieldsRead;
267
268	/** Whether the guest debug state was active at the time of VM-exit. */
269	bool fWasGuestDebugStateActive;
270	/** Whether the hyper debug state was active at the time of VM-exit. */
271	bool fWasHyperDebugStateActive;
272	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
273	bool fUpdatedTscOffsettingAndPreemptTimer;
274	/** Whether the VM-exit was caused by a page-fault during delivery of a
275	* contributory exception or a page-fault. */
276	bool fVectoringDoublePF;
277	/** Whether the VM-exit was caused by a page-fault during delivery of an
278	* external interrupt or NMI. */
279	bool fVectoringPF;
280	bool afAlignment0[3];
281
282	/** The VMCS info. object. */
283	PVMXVMCSINFO pVmcsInfo;
284	} VMXTRANSIENT;
285	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
286	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
287	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
288	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
289	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
290	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
291	/** Pointer to VMX transient state. */
292	typedef VMXTRANSIENT *PVMXTRANSIENT;
293
294	/**
295	* Memory operand read or write access.
296	*/
297	typedef enum VMXMEMACCESS
298	{
299	VMXMEMACCESS_READ = 0,
300	VMXMEMACCESS_WRITE = 1
301	} VMXMEMACCESS;
302
303	/**
304	* VMX VM-exit handler.
305	*
306	* @returns Strict VBox status code (i.e. informational status codes too).
307	* @param pVCpu The cross context virtual CPU structure.
308	* @param pVmxTransient The VMX-transient structure.
309	*/
310	#ifndef HMVMX_USE_FUNCTION_TABLE
311	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
312	#else
313	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
314	/** Pointer to VM-exit handler. */
315	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
316	#endif
317
318	/**
319	* VMX VM-exit handler, non-strict status code.
320	*
321	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
322	*
323	* @returns VBox status code, no informational status code returned.
324	* @param pVCpu The cross context virtual CPU structure.
325	* @param pVmxTransient The VMX-transient structure.
326	*
327	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
328	* use of that status code will be replaced with VINF_EM_SOMETHING
329	* later when switching over to IEM.
330	*/
331	#ifndef HMVMX_USE_FUNCTION_TABLE
332	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
333	#else
334	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
335	#endif
336
337
338	/*********************************************************************************************************************************
339	* Internal Functions *
340	*********************************************************************************************************************************/
341	#ifndef HMVMX_USE_FUNCTION_TABLE
342	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
343	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
344	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
345	#else
346	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
347	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
348	#endif
349	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
350	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
351	#endif
352
353	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
354	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
355	static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
356	#endif
357
358	/** @name VM-exit handlers.
359	* @{
360	*/
361	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
362	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
363	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
364	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
365	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
366	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
367	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
368	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
369	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
370	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
371	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
372	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
373	static FNVMXEXITHANDLER hmR0VmxExitHlt;
374	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
375	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
376	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
377	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
378	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
379	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
380	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
381	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
382	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
383	static FNVMXEXITHANDLER hmR0VmxExitVmread;
384	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
385	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
386	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
387	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
388	#endif
389	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
390	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
391	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
392	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
393	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
394	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
395	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
396	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
397	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
398	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
399	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
400	static FNVMXEXITHANDLER hmR0VmxExitMwait;
401	static FNVMXEXITHANDLER hmR0VmxExitMtf;
402	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
403	static FNVMXEXITHANDLER hmR0VmxExitPause;
404	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
405	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
406	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
407	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
408	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
409	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
410	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
411	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
412	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
413	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
414	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
415	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
416	/** @} */
417
418	/** @name Helpers for hardware exceptions VM-exit handlers.
419	* @{
420	*/
421	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
422	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
423	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
424	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
425	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
426	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
427	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
428	/** @} */
429
430
431	/*********************************************************************************************************************************
432	* Global Variables *
433	*********************************************************************************************************************************/
434	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
435	static const uint32_t g_aVmcsCacheSegBase[] =
436	{
437	VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
438	VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
439	VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
440	VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
441	VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
442	VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
443	};
444	AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
445	#endif
446	static const uint32_t g_aVmcsSegBase[] =
447	{
448	VMX_VMCS_GUEST_ES_BASE,
449	VMX_VMCS_GUEST_CS_BASE,
450	VMX_VMCS_GUEST_SS_BASE,
451	VMX_VMCS_GUEST_DS_BASE,
452	VMX_VMCS_GUEST_FS_BASE,
453	VMX_VMCS_GUEST_GS_BASE
454	};
455	static const uint32_t g_aVmcsSegSel[] =
456	{
457	VMX_VMCS16_GUEST_ES_SEL,
458	VMX_VMCS16_GUEST_CS_SEL,
459	VMX_VMCS16_GUEST_SS_SEL,
460	VMX_VMCS16_GUEST_DS_SEL,
461	VMX_VMCS16_GUEST_FS_SEL,
462	VMX_VMCS16_GUEST_GS_SEL
463	};
464	static const uint32_t g_aVmcsSegLimit[] =
465	{
466	VMX_VMCS32_GUEST_ES_LIMIT,
467	VMX_VMCS32_GUEST_CS_LIMIT,
468	VMX_VMCS32_GUEST_SS_LIMIT,
469	VMX_VMCS32_GUEST_DS_LIMIT,
470	VMX_VMCS32_GUEST_FS_LIMIT,
471	VMX_VMCS32_GUEST_GS_LIMIT
472	};
473	static const uint32_t g_aVmcsSegAttr[] =
474	{
475	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
476	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
477	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
478	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
479	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
480	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
481	};
482	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
483	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
484	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
485	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
486
487	#ifdef HMVMX_USE_FUNCTION_TABLE
488	/**
489	* VMX_EXIT dispatch table.
490	*/
491	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
492	{
493	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
494	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
495	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
496	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
497	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
498	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
499	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
500	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
501	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
502	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
503	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
504	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
505	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
506	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
507	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
508	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
509	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
510	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
511	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
512	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
513	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
514	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
515	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
516	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
517	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
518	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
519	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
520	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
521	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
522	#else
523	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
524	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
525	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
526	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
527	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
528	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
529	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
530	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
531	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
532	#endif
533	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
534	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
535	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
536	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
537	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
538	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
539	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
540	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
541	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
542	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
543	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
544	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
545	/* 40 UNDEFINED */ hmR0VmxExitPause,
546	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
547	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
548	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
549	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
550	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
551	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitXdtrAccess,
552	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitXdtrAccess,
553	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
554	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
555	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
556	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
557	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
558	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
559	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
560	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
561	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
562	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
563	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
564	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
565	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
566	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
567	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
568	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
569	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
570	};
571	#endif /* HMVMX_USE_FUNCTION_TABLE */
572
573	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
574	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
575	{
576	/* 0 */ "(Not Used)",
577	/* 1 */ "VMCALL executed in VMX root operation.",
578	/* 2 */ "VMCLEAR with invalid physical address.",
579	/* 3 */ "VMCLEAR with VMXON pointer.",
580	/* 4 */ "VMLAUNCH with non-clear VMCS.",
581	/* 5 */ "VMRESUME with non-launched VMCS.",
582	/* 6 */ "VMRESUME after VMXOFF",
583	/* 7 */ "VM-entry with invalid control fields.",
584	/* 8 */ "VM-entry with invalid host state fields.",
585	/* 9 */ "VMPTRLD with invalid physical address.",
586	/* 10 */ "VMPTRLD with VMXON pointer.",
587	/* 11 */ "VMPTRLD with incorrect revision identifier.",
588	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
589	/* 13 */ "VMWRITE to read-only VMCS component.",
590	/* 14 */ "(Not Used)",
591	/* 15 */ "VMXON executed in VMX root operation.",
592	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
593	/* 17 */ "VM-entry with non-launched executing VMCS.",
594	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
595	/* 19 */ "VMCALL with non-clear VMCS.",
596	/* 20 */ "VMCALL with invalid VM-exit control fields.",
597	/* 21 */ "(Not Used)",
598	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
599	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
600	/* 24 */ "VMCALL with invalid SMM-monitor features.",
601	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
602	/* 26 */ "VM-entry with events blocked by MOV SS.",
603	/* 27 */ "(Not Used)",
604	/* 28 */ "Invalid operand to INVEPT/INVVPID."
605	};
606	#endif /* VBOX_STRICT */
607
608
609	/**
610	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
611	*
612	* Any bit set in this mask is owned by the host/hypervisor and would cause a
613	* VM-exit when modified by the guest.
614	*
615	* @returns The static CR0 guest/host mask.
616	* @param pVCpu The cross context virtual CPU structure.
617	*/
618	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PVMCPU pVCpu)
619	{
620	/*
621	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
622	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
623	*/
624	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
625	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
626	* and @bugref{6944}. */
627	PVM pVM = pVCpu->CTX_SUFF(pVM);
628	return ( X86_CR0_PE
629	\| X86_CR0_NE
630	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
631	\| X86_CR0_PG
632	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
633	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
634	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
635	}
636
637
638	/**
639	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
640	*
641	* Any bit set in this mask is owned by the host/hypervisor and would cause a
642	* VM-exit when modified by the guest.
643	*
644	* @returns The static CR4 guest/host mask.
645	* @param pVCpu The cross context virtual CPU structure.
646	*/
647	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PVMCPU pVCpu)
648	{
649	/*
650	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
651	* these bits are reserved on hardware that does not support them. Since the
652	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
653	* these bits and handle it depending on whether we expose them to the guest.
654	*/
655	PVM pVM = pVCpu->CTX_SUFF(pVM);
656	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
657	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
658	return ( X86_CR4_VMXE
659	\| X86_CR4_VME
660	\| X86_CR4_PAE
661	\| X86_CR4_PGE
662	\| X86_CR4_PSE
663	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
664	\| (fPcid ? X86_CR4_PCIDE : 0));
665	}
666
667
668	/**
669	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
670	* area.
671	*
672	* @returns @c true if it's different, @c false otherwise.
673	* @param pVmcsInfo The VMCS info. object.
674	*/
675	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
676	{
677	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
678	&& pVmcsInfo->pvGuestMsrStore);
679	}
680
681
682	/**
683	* Adds one or more exceptions to the exception bitmap and commits it to the current
684	* VMCS.
685	*
686	* @returns VBox status code.
687	* @param pVmxTransient The VMX-transient structure.
688	* @param uXcptMask The exception(s) to add.
689	*/
690	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
691	{
692	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
693	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
694	if ((uXcptBitmap & uXcptMask) != uXcptMask)
695	{
696	uXcptBitmap \|= uXcptMask;
697	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
698	AssertRCReturn(rc, rc);
699	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
700	}
701	return VINF_SUCCESS;
702	}
703
704
705	/**
706	* Adds an exception to the exception bitmap and commits it to the current VMCS.
707	*
708	* @returns VBox status code.
709	* @param pVmxTransient The VMX-transient structure.
710	* @param uXcpt The exception to add.
711	*/
712	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
713	{
714	Assert(uXcpt <= X86_XCPT_LAST);
715	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
716	}
717
718
719	/**
720	* Remove one or more exceptions from the exception bitmap and commits it to the
721	* current VMCS.
722	*
723	* This takes care of not removing the exception intercept if a nested-guest
724	* requires the exception to be intercepted.
725	*
726	* @returns VBox status code.
727	* @param pVCpu The cross context virtual CPU structure.
728	* @param pVmxTransient The VMX-transient structure.
729	* @param uXcptMask The exception(s) to remove.
730	*/
731	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
732	{
733	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
734	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
735	if (u32XcptBitmap & uXcptMask)
736	{
737	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
738	if (!pVmxTransient->fIsNestedGuest)
739	{ /* likely */ }
740	else
741	{
742	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
743	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
744	}
745	#endif
746	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
747	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
748	\| RT_BIT(X86_XCPT_DE)
749	\| RT_BIT(X86_XCPT_NM)
750	\| RT_BIT(X86_XCPT_TS)
751	\| RT_BIT(X86_XCPT_UD)
752	\| RT_BIT(X86_XCPT_NP)
753	\| RT_BIT(X86_XCPT_SS)
754	\| RT_BIT(X86_XCPT_GP)
755	\| RT_BIT(X86_XCPT_PF)
756	\| RT_BIT(X86_XCPT_MF));
757	#elif defined(HMVMX_ALWAYS_TRAP_PF)
758	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
759	#endif
760	if (uXcptMask)
761	{
762	/* Validate we are not removing any essential exception intercepts. */
763	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
764	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
765	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
766
767	/* Remove it from the exception bitmap. */
768	u32XcptBitmap &= ~uXcptMask;
769
770	/* Commit and update the cache if necessary. */
771	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
772	{
773	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
774	AssertRCReturn(rc, rc);
775	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
776	}
777	}
778	}
779	return VINF_SUCCESS;
780	}
781
782
783	/**
784	* Remove an exceptions from the exception bitmap and commits it to the current
785	* VMCS.
786	*
787	* @returns VBox status code.
788	* @param pVCpu The cross context virtual CPU structure.
789	* @param pVmxTransient The VMX-transient structure.
790	* @param uXcpt The exception to remove.
791	*/
792	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
793	{
794	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
795	}
796
797
798	/**
799	* Loads the VMCS specified by the VMCS info. object.
800	*
801	* @returns VBox status code.
802	* @param pVmcsInfo The VMCS info. object.
803	*/
804	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
805	{
806	Assert(pVmcsInfo);
807	Assert(pVmcsInfo->HCPhysVmcs);
808	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
809
810	if (pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_CLEAR)
811	{
812	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
813	if (RT_SUCCESS(rc))
814	{
815	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
816	return VINF_SUCCESS;
817	}
818	return rc;
819	}
820	return VERR_VMX_INVALID_VMCS_LAUNCH_STATE;
821	}
822
823
824	/**
825	* Clears the VMCS specified by the VMCS info. object.
826	*
827	* @returns VBox status code.
828	* @param pVmcsInfo The VMCS info. object.
829	*/
830	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
831	{
832	Assert(pVmcsInfo);
833	Assert(pVmcsInfo->HCPhysVmcs);
834	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
835
836	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
837	if (RT_SUCCESS(rc))
838	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
839	return rc;
840	}
841
842
843	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
844	/**
845	* Switches the current VMCS to the one specified.
846	*
847	* @returns VBox status code.
848	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
849	* @param pVmcsInfoTo The VMCS info. object we are switching to.
850	*
851	* @remarks Called with interrupts disabled.
852	*/
853	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
854	{
855	Assert(pVmcsInfoFrom);
856	Assert(pVmcsInfoTo);
857
858	/*
859	* Clear the VMCS we are switching out if it has not already been cleared.
860	* This will sync any CPU internal data back to the VMCS.
861	*/
862	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
863	{
864	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
865	if (RT_SUCCESS(rc))
866	{ /* likely */ }
867	else
868	return rc;
869	}
870
871	/*
872	* Clear the VMCS we are switching to if it has not already been cleared.
873	* This will initialize the VMCS launch state to "clear" required for loading it.
874	*
875	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
876	*/
877	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
878	{
879	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
880	if (RT_SUCCESS(rc))
881	{ /* likely */ }
882	else
883	return rc;
884	}
885
886	/*
887	* Finally, load the VMCS we are switching to.
888	*/
889	return hmR0VmxLoadVmcs(pVmcsInfoTo);
890	}
891	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
892
893
894	/**
895	* Updates the VM's last error record.
896	*
897	* If there was a VMX instruction error, reads the error data from the VMCS and
898	* updates VCPU's last error record as well.
899	*
900	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
901	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
902	* VERR_VMX_INVALID_VMCS_FIELD.
903	* @param rc The error code.
904	*/
905	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
906	{
907	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
908	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
909	{
910	AssertPtrReturnVoid(pVCpu);
911	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
912	}
913	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
914	}
915
916
917	/**
918	* Reads the VM-entry interruption-information field from the VMCS into the VMX
919	* transient structure.
920	*
921	* @returns VBox status code.
922	* @param pVmxTransient The VMX-transient structure.
923	*
924	* @remarks No-long-jump zone!!!
925	*/
926	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
927	{
928	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
929	AssertRCReturn(rc, rc);
930	return VINF_SUCCESS;
931	}
932
933
934	#ifdef VBOX_STRICT
935	/**
936	* Reads the VM-entry exception error code field from the VMCS into
937	* the VMX transient structure.
938	*
939	* @returns VBox status code.
940	* @param pVmxTransient The VMX-transient structure.
941	*
942	* @remarks No-long-jump zone!!!
943	*/
944	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
945	{
946	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
947	AssertRCReturn(rc, rc);
948	return VINF_SUCCESS;
949	}
950
951
952	/**
953	* Reads the VM-entry exception error code field from the VMCS into
954	* the VMX transient structure.
955	*
956	* @returns VBox status code.
957	* @param pVmxTransient The VMX-transient structure.
958	*
959	* @remarks No-long-jump zone!!!
960	*/
961	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
962	{
963	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
964	AssertRCReturn(rc, rc);
965	return VINF_SUCCESS;
966	}
967	#endif /* VBOX_STRICT */
968
969
970	/**
971	* Reads the VM-exit interruption-information field from the VMCS into the VMX
972	* transient structure.
973	*
974	* @returns VBox status code.
975	* @param pVmxTransient The VMX-transient structure.
976	*/
977	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
978	{
979	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
980	{
981	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
982	AssertRCReturn(rc,rc);
983	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
984	}
985	return VINF_SUCCESS;
986	}
987
988
989	/**
990	* Reads the VM-exit interruption error code from the VMCS into the VMX
991	* transient structure.
992	*
993	* @returns VBox status code.
994	* @param pVmxTransient The VMX-transient structure.
995	*/
996	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
997	{
998	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
999	{
1000	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1001	AssertRCReturn(rc, rc);
1002	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1003	}
1004	return VINF_SUCCESS;
1005	}
1006
1007
1008	/**
1009	* Reads the VM-exit instruction length field from the VMCS into the VMX
1010	* transient structure.
1011	*
1012	* @returns VBox status code.
1013	* @param pVmxTransient The VMX-transient structure.
1014	*/
1015	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1016	{
1017	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1018	{
1019	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1020	AssertRCReturn(rc, rc);
1021	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1022	}
1023	return VINF_SUCCESS;
1024	}
1025
1026
1027	/**
1028	* Reads the VM-exit instruction-information field from the VMCS into
1029	* the VMX transient structure.
1030	*
1031	* @returns VBox status code.
1032	* @param pVmxTransient The VMX-transient structure.
1033	*/
1034	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1035	{
1036	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1037	{
1038	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1039	AssertRCReturn(rc, rc);
1040	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1041	}
1042	return VINF_SUCCESS;
1043	}
1044
1045
1046	/**
1047	* Reads the VM-exit Qualification from the VMCS into the VMX transient structure.
1048	*
1049	* @returns VBox status code.
1050	* @param pVCpu The cross context virtual CPU structure of the
1051	* calling EMT. (Required for the VMCS cache case.)
1052	* @param pVmxTransient The VMX-transient structure.
1053	*/
1054	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1055	{
1056	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1057	{
1058	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1059	AssertRCReturn(rc, rc);
1060	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1061	}
1062	return VINF_SUCCESS;
1063	}
1064
1065
1066	/**
1067	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1068	*
1069	* @returns VBox status code.
1070	* @param pVCpu The cross context virtual CPU structure of the
1071	* calling EMT. (Required for the VMCS cache case.)
1072	* @param pVmxTransient The VMX-transient structure.
1073	*/
1074	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1075	{
1076	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1077	{
1078	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1079	AssertRCReturn(rc, rc);
1080	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1081	}
1082	return VINF_SUCCESS;
1083	}
1084
1085
1086	/**
1087	* Reads the IDT-vectoring information field from the VMCS into the VMX
1088	* transient structure.
1089	*
1090	* @returns VBox status code.
1091	* @param pVmxTransient The VMX-transient structure.
1092	*
1093	* @remarks No-long-jump zone!!!
1094	*/
1095	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1096	{
1097	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1098	{
1099	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1100	AssertRCReturn(rc, rc);
1101	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1102	}
1103	return VINF_SUCCESS;
1104	}
1105
1106
1107	/**
1108	* Reads the IDT-vectoring error code from the VMCS into the VMX
1109	* transient structure.
1110	*
1111	* @returns VBox status code.
1112	* @param pVmxTransient The VMX-transient structure.
1113	*/
1114	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1115	{
1116	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1117	{
1118	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1119	AssertRCReturn(rc, rc);
1120	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1121	}
1122	return VINF_SUCCESS;
1123	}
1124
1125
1126	/**
1127	* Enters VMX root mode operation on the current CPU.
1128	*
1129	* @returns VBox status code.
1130	* @param pVM The cross context VM structure. Can be
1131	* NULL, after a resume.
1132	* @param HCPhysCpuPage Physical address of the VMXON region.
1133	* @param pvCpuPage Pointer to the VMXON region.
1134	*/
1135	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1136	{
1137	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1138	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1139	Assert(pvCpuPage);
1140	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1141
1142	if (pVM)
1143	{
1144	/* Write the VMCS revision identifier to the VMXON region. */
1145	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1146	}
1147
1148	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1149	RTCCUINTREG fEFlags = ASMIntDisableFlags();
1150
1151	/* Enable the VMX bit in CR4 if necessary. */
1152	RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1153
1154	/* Enter VMX root mode. */
1155	int rc = VMXEnable(HCPhysCpuPage);
1156	if (RT_FAILURE(rc))
1157	{
1158	if (!(uOldCr4 & X86_CR4_VMXE))
1159	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
1160
1161	if (pVM)
1162	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1163	}
1164
1165	/* Restore interrupts. */
1166	ASMSetFlags(fEFlags);
1167	return rc;
1168	}
1169
1170
1171	/**
1172	* Exits VMX root mode operation on the current CPU.
1173	*
1174	* @returns VBox status code.
1175	*/
1176	static int hmR0VmxLeaveRootMode(void)
1177	{
1178	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1179
1180	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1181	RTCCUINTREG fEFlags = ASMIntDisableFlags();
1182
1183	/* If we're for some reason not in VMX root mode, then don't leave it. */
1184	RTCCUINTREG uHostCR4 = ASMGetCR4();
1185
1186	int rc;
1187	if (uHostCR4 & X86_CR4_VMXE)
1188	{
1189	/* Exit VMX root mode and clear the VMX bit in CR4. */
1190	VMXDisable();
1191	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
1192	rc = VINF_SUCCESS;
1193	}
1194	else
1195	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1196
1197	/* Restore interrupts. */
1198	ASMSetFlags(fEFlags);
1199	return rc;
1200	}
1201
1202
1203	/**
1204	* Allocates and maps a physically contiguous page. The allocated page is
1205	* zero'd out (used by various VT-x structures).
1206	*
1207	* @returns IPRT status code.
1208	* @param pMemObj Pointer to the ring-0 memory object.
1209	* @param ppVirt Where to store the virtual address of the
1210	* allocation.
1211	* @param pHCPhys Where to store the physical address of the
1212	* allocation.
1213	*/
1214	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1215	{
1216	AssertPtr(pMemObj);
1217	AssertPtr(ppVirt);
1218	AssertPtr(pHCPhys);
1219	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1220	if (RT_FAILURE(rc))
1221	return rc;
1222	ppVirt = RTR0MemObjAddress(pMemObj);
1223	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1224	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1225	return VINF_SUCCESS;
1226	}
1227
1228
1229	/**
1230	* Frees and unmaps an allocated, physical page.
1231	*
1232	* @param pMemObj Pointer to the ring-0 memory object.
1233	* @param ppVirt Where to re-initialize the virtual address of
1234	* allocation as 0.
1235	* @param pHCPhys Where to re-initialize the physical address of the
1236	* allocation as 0.
1237	*/
1238	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1239	{
1240	AssertPtr(pMemObj);
1241	AssertPtr(ppVirt);
1242	AssertPtr(pHCPhys);
1243	/* NULL is valid, accepted and ignored by the free function below. */
1244	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1245	*pMemObj = NIL_RTR0MEMOBJ;
1246	*ppVirt = NULL;
1247	*pHCPhys = NIL_RTHCPHYS;
1248	}
1249
1250
1251	/**
1252	* Initializes a VMCS info. object.
1253	*
1254	* @param pVmcsInfo The VMCS info. object.
1255	*/
1256	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1257	{
1258	RT_ZERO(*pVmcsInfo);
1259
1260	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1261	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1262	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1263	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1264	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1265	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1266	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1267	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1268	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1269	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1270	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1271	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1272	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1273	}
1274
1275
1276	/**
1277	* Frees the VT-x structures for a VMCS info. object.
1278	*
1279	* @param pVM The cross context VM structure.
1280	* @param pVmcsInfo The VMCS info. object.
1281	*/
1282	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1283	{
1284	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1285
1286	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1287	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1288
1289	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1290	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1291	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1292
1293	hmR0VmxInitVmcsInfo(pVmcsInfo);
1294	}
1295
1296
1297	/**
1298	* Allocates the VT-x structures for a VMCS info. object.
1299	*
1300	* @returns VBox status code.
1301	* @param pVCpu The cross context virtual CPU structure.
1302	* @param pVmcsInfo The VMCS info. object.
1303	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1304	*/
1305	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1306	{
1307	PVM pVM = pVCpu->CTX_SUFF(pVM);
1308
1309	/* Allocate the guest VM control structure (VMCS). */
1310	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1311	if (RT_SUCCESS(rc))
1312	{
1313	if (!fIsNstGstVmcs)
1314	{
1315	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1316	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1317	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1318	{
1319	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1320	NULL /* pR3Ptr /, NULL / pRCPtr */);
1321	}
1322	}
1323	else
1324	{
1325	Assert(pVmcsInfo->HCPhysVirtApic == NIL_RTHCPHYS);
1326	Assert(!pVmcsInfo->pbVirtApic);
1327	}
1328
1329	if (RT_SUCCESS(rc))
1330	{
1331	/*
1332	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1333	* transparent accesses of specific MSRs.
1334	*
1335	* If the condition for enabling MSR bitmaps changes here, don't forget to
1336	* update HMIsMsrBitmapActive().
1337	*
1338	* We don't share MSR bitmaps between the guest and nested-guest as we then
1339	* don't need to care about carefully restoring the guest MSR bitmap.
1340	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1341	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1342	*/
1343	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1344	{
1345	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1346	if (RT_SUCCESS(rc))
1347	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1348	}
1349
1350	if (RT_SUCCESS(rc))
1351	{
1352	/*
1353	* Allocate the VM-entry MSR-load area for the guest MSRs.
1354	*
1355	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1356	* the guest and nested-guest.
1357	*/
1358	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1359	&pVmcsInfo->HCPhysGuestMsrLoad);
1360	if (RT_SUCCESS(rc))
1361	{
1362	/*
1363	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1364	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1365	*/
1366	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1367	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1368	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1369
1370	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1371	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1372	&pVmcsInfo->HCPhysHostMsrLoad);
1373	}
1374	}
1375	}
1376	}
1377
1378	return rc;
1379	}
1380
1381
1382	/**
1383	* Free all VT-x structures for the VM.
1384	*
1385	* @returns IPRT status code.
1386	* @param pVM The cross context VM structure.
1387	*/
1388	static void hmR0VmxStructsFree(PVM pVM)
1389	{
1390	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1391	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1392	#endif
1393	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1394
1395	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1396	{
1397	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1398	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1399	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1400	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1401	if (pVM->cpum.ro.GuestFeatures.fVmx)
1402	{
1403	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1404	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1405	}
1406	#endif
1407	}
1408	}
1409
1410
1411	/**
1412	* Allocate all VT-x structures for the VM.
1413	*
1414	* @returns IPRT status code.
1415	* @param pVM The cross context VM structure.
1416	*/
1417	static int hmR0VmxStructsAlloc(PVM pVM)
1418	{
1419	/*
1420	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1421	* The VMCS size cannot be more than 4096 bytes.
1422	*
1423	* See Intel spec. Appendix A.1 "Basic VMX Information".
1424	*/
1425	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1426	if (cbVmcs <= X86_PAGE_4K_SIZE)
1427	{ /* likely */ }
1428	else
1429	{
1430	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1431	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1432	}
1433
1434	/*
1435	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1436	*/
1437	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1438	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1439	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1440	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1441	#endif
1442
1443	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1444	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1445	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1446
1447	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1448	{
1449	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1450	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1451	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1452	}
1453
1454	/*
1455	* Allocate per-VM VT-x structures.
1456	*/
1457	int rc = VINF_SUCCESS;
1458	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1459	/* Allocate crash-dump magic scratch page. */
1460	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1461	if (RT_FAILURE(rc))
1462	{
1463	hmR0VmxStructsFree(pVM);
1464	return rc;
1465	}
1466	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1467	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1468	#endif
1469
1470	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1471	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1472	{
1473	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1474	&pVM->hm.s.vmx.HCPhysApicAccess);
1475	if (RT_FAILURE(rc))
1476	{
1477	hmR0VmxStructsFree(pVM);
1478	return rc;
1479	}
1480	}
1481
1482	/*
1483	* Initialize per-VCPU VT-x structures.
1484	*/
1485	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1486	{
1487	/* Allocate the guest VMCS structures. */
1488	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1489	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
1490	if (RT_SUCCESS(rc))
1491	{
1492	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1493	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
1494	if (pVM->cpum.ro.GuestFeatures.fVmx)
1495	{
1496	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
1497	if (RT_SUCCESS(rc))
1498	{ /* likely */ }
1499	else
1500	break;
1501	}
1502	#endif
1503	}
1504	else
1505	break;
1506	}
1507
1508	if (RT_FAILURE(rc))
1509	{
1510	hmR0VmxStructsFree(pVM);
1511	return rc;
1512	}
1513
1514	return VINF_SUCCESS;
1515	}
1516
1517
1518	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1519	/**
1520	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
1521	*
1522	* @returns @c true if the MSR is intercepted, @c false otherwise.
1523	* @param pvMsrBitmap The MSR bitmap.
1524	* @param offMsr The MSR byte offset.
1525	* @param iBit The bit offset from the byte offset.
1526	*/
1527	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
1528	{
1529	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
1530	Assert(pbMsrBitmap);
1531	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
1532	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
1533	}
1534	#endif
1535
1536
1537	/**
1538	* Sets the permission bits for the specified MSR in the given MSR bitmap.
1539	*
1540	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
1541	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
1542	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
1543	* the read/write access of this MSR.
1544	*
1545	* @param pVCpu The cross context virtual CPU structure.
1546	* @param pVmcsInfo The VMCS info. object.
1547	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1548	* @param idMsr The MSR value.
1549	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
1550	* include both a read -and- a write permission!
1551	*
1552	* @sa HMGetVmxMsrPermission.
1553	*/
1554	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
1555	{
1556	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
1557	Assert(pbMsrBitmap);
1558	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
1559
1560	/*
1561	* MSR-bitmap Layout:
1562	* Byte index MSR range Interpreted as
1563	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1564	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1565	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1566	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1567	*
1568	* A bit corresponding to an MSR within the above range causes a VM-exit
1569	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
1570	* the MSR range, it always cause a VM-exit.
1571	*
1572	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
1573	*/
1574	uint16_t const offBitmapRead = 0;
1575	uint16_t const offBitmapWrite = 0x800;
1576	uint16_t offMsr;
1577	int32_t iBit;
1578	if (idMsr <= UINT32_C(0x00001fff))
1579	{
1580	offMsr = 0;
1581	iBit = idMsr;
1582	}
1583	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1584	{
1585	offMsr = 0x400;
1586	iBit = idMsr - UINT32_C(0xc0000000);
1587	}
1588	else
1589	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
1590
1591	/*
1592	* Set the MSR read permission.
1593	*/
1594	uint16_t const offMsrRead = offBitmapRead + offMsr;
1595	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
1596	if (fMsrpm & VMXMSRPM_ALLOW_RD)
1597	{
1598	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1599	bool const fClear = !fIsNstGstVmcs ? true
1600	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
1601	#else
1602	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1603	bool const fClear = true;
1604	#endif
1605	if (fClear)
1606	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
1607	}
1608	else
1609	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
1610
1611	/*
1612	* Set the MSR write permission.
1613	*/
1614	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
1615	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
1616	if (fMsrpm & VMXMSRPM_ALLOW_WR)
1617	{
1618	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1619	bool const fClear = !fIsNstGstVmcs ? true
1620	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
1621	#else
1622	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1623	bool const fClear = true;
1624	#endif
1625	if (fClear)
1626	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
1627	}
1628	else
1629	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
1630	}
1631
1632
1633	/**
1634	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1635	* area.
1636	*
1637	* @returns VBox status code.
1638	* @param pVCpu The cross context virtual CPU structure.
1639	* @param pVmcsInfo The VMCS info. object.
1640	* @param cMsrs The number of MSRs.
1641	*/
1642	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
1643	{
1644	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1645	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1646	if (RT_UNLIKELY(cMsrs >= cMaxSupportedMsrs))
1647	{
1648	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1649	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1650	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1651	}
1652
1653	/* Commit the MSR counts to the VMCS and update the cache. */
1654	int rc = VINF_SUCCESS;
1655	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
1656	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1657	if (pVmcsInfo->cExitMsrStore != cMsrs)
1658	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1659	if (pVmcsInfo->cExitMsrLoad != cMsrs)
1660	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1661	AssertRCReturn(rc, rc);
1662
1663	pVmcsInfo->cEntryMsrLoad = cMsrs;
1664	pVmcsInfo->cExitMsrStore = cMsrs;
1665	pVmcsInfo->cExitMsrLoad = cMsrs;
1666
1667	return VINF_SUCCESS;
1668	}
1669
1670
1671	/**
1672	* Adds a new (or updates the value of an existing) guest/host MSR
1673	* pair to be swapped during the world-switch as part of the
1674	* auto-load/store MSR area in the VMCS.
1675	*
1676	* @returns VBox status code.
1677	* @param pVCpu The cross context virtual CPU structure.
1678	* @param pVmxTransient The VMX-transient structure.
1679	* @param idMsr The MSR.
1680	* @param uGuestMsrValue Value of the guest MSR.
1681	* @param fSetReadWrite Whether to set the guest read/write access of this
1682	* MSR (thus not causing a VM-exit).
1683	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1684	* necessary.
1685	*/
1686	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
1687	bool fSetReadWrite, bool fUpdateHostMsr)
1688	{
1689	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1690	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1691	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1692	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1693	uint32_t i;
1694
1695	/* Paranoia. */
1696	Assert(pGuestMsrLoad);
1697
1698	/* Check if the MSR already exists in the VM-entry MSR-load area. */
1699	for (i = 0; i < cMsrs; i++)
1700	{
1701	if (pGuestMsrLoad->u32Msr == idMsr)
1702	break;
1703	pGuestMsrLoad++;
1704	}
1705
1706	bool fAdded = false;
1707	if (i == cMsrs)
1708	{
1709	/* The MSR does not exist, bump the MSR coun to make room for the new MSR. */
1710	++cMsrs;
1711	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1712	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
1713
1714	/* Set the guest to read/write this MSR without causing VM-exits. */
1715	if ( fSetReadWrite
1716	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
1717	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
1718
1719	fAdded = true;
1720	}
1721
1722	/* Update the MSR value for the newly added or already existing MSR. */
1723	pGuestMsrLoad->u32Msr = idMsr;
1724	pGuestMsrLoad->u64Value = uGuestMsrValue;
1725
1726	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
1727	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1728	{
1729	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1730	pGuestMsrStore += i;
1731	pGuestMsrStore->u32Msr = idMsr;
1732	pGuestMsrStore->u64Value = 0;
1733	}
1734
1735	/* Update the corresponding slot in the host MSR area. */
1736	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1737	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad && pHostMsr != pVmcsInfo->pvGuestMsrStore);
1738	pHostMsr += i;
1739	pHostMsr->u32Msr = idMsr;
1740
1741	/*
1742	* Only if the caller requests to update the host MSR value AND we've newly added the
1743	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
1744	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
1745	*
1746	* We do this for performance reasons since reading MSRs may be quite expensive.
1747	*/
1748	if ( fAdded
1749	&& fUpdateHostMsr)
1750	{
1751	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1752	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1753	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1754	}
1755	return VINF_SUCCESS;
1756	}
1757
1758
1759	/**
1760	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1761	* auto-load/store MSR area in the VMCS.
1762	*
1763	* @returns VBox status code.
1764	* @param pVCpu The cross context virtual CPU structure.
1765	* @param pVmxTransient The VMX-transient structure.
1766	* @param idMsr The MSR.
1767	*/
1768	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
1769	{
1770	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1771	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1772	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1773	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1774
1775	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
1776	for (uint32_t i = 0; i < cMsrs; i++)
1777	{
1778	/* Find the MSR. */
1779	if (pGuestMsrLoad->u32Msr == idMsr)
1780	{
1781	/* If it's the last MSR, simply reduce the count. */
1782	if (i == cMsrs - 1)
1783	{
1784	--cMsrs;
1785	break;
1786	}
1787
1788	/* Remove it by copying the last MSR in place of it, and reducing the count. */
1789	PVMXAUTOMSR pLastGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1790	pLastGuestMsrLoad += cMsrs - 1;
1791	pGuestMsrLoad->u32Msr = pLastGuestMsrLoad->u32Msr;
1792	pGuestMsrLoad->u64Value = pLastGuestMsrLoad->u64Value;
1793
1794	/* Remove it from the VM-exit MSR-store area if we are using a different page. */
1795	if (fSeparateExitMsrStorePage)
1796	{
1797	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1798	PVMXAUTOMSR pLastGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1799	pGuestMsrStore += i;
1800	pLastGuestMsrStore += cMsrs - 1;
1801	Assert(pGuestMsrStore->u32Msr == idMsr);
1802	pGuestMsrStore->u32Msr = pLastGuestMsrStore->u32Msr;
1803	pGuestMsrStore->u64Value = pLastGuestMsrStore->u64Value;
1804	}
1805
1806	/* Remove it from the VM-exit MSR-load area. */
1807	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1808	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1809	pHostMsr += i;
1810	pLastHostMsr += cMsrs - 1;
1811	Assert(pHostMsr->u32Msr == idMsr);
1812	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1813	pHostMsr->u64Value = pLastHostMsr->u64Value;
1814	--cMsrs;
1815	break;
1816	}
1817	pGuestMsrLoad++;
1818	}
1819
1820	/* Update the VMCS if the count changed (meaning the MSR was found). */
1821	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
1822	{
1823	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1824	AssertRCReturn(rc, rc);
1825
1826	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1827	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1828	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
1829
1830	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
1831	return VINF_SUCCESS;
1832	}
1833
1834	return VERR_NOT_FOUND;
1835	}
1836
1837
1838	/**
1839	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
1840	*
1841	* @returns @c true if found, @c false otherwise.
1842	* @param pVmcsInfo The VMCS info. object.
1843	* @param idMsr The MSR to find.
1844	*/
1845	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
1846	{
1847	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1848	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
1849	for (uint32_t i = 0; i < cMsrs; i++)
1850	{
1851	if (pGuestMsrLoad->u32Msr == idMsr)
1852	return true;
1853	pGuestMsrLoad++;
1854	}
1855	return false;
1856	}
1857
1858
1859	/**
1860	* Updates the value of all host MSRs in the VM-exit MSR-load area.
1861	*
1862	* @param pVCpu The cross context virtual CPU structure.
1863	* @param pVmcsInfo The VMCS info. object.
1864	*
1865	* @remarks No-long-jump zone!!!
1866	*/
1867	static void hmR0VmxUpdateAutoLoadHostMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
1868	{
1869	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1870	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
1871
1872	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1873	Assert(pHostMsrLoad);
1874
1875	for (uint32_t i = 0; i < cMsrs; i++, pHostMsrLoad++)
1876	{
1877	/*
1878	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1879	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1880	*/
1881	if (pHostMsrLoad->u32Msr == MSR_K6_EFER)
1882	pHostMsrLoad->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
1883	else
1884	pHostMsrLoad->u64Value = ASMRdMsr(pHostMsrLoad->u32Msr);
1885	}
1886	}
1887
1888
1889	/**
1890	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1891	* perform lazy restoration of the host MSRs while leaving VT-x.
1892	*
1893	* @param pVCpu The cross context virtual CPU structure.
1894	*
1895	* @remarks No-long-jump zone!!!
1896	*/
1897	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1898	{
1899	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1900
1901	/*
1902	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
1903	*/
1904	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1905	{
1906	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1907	#if HC_ARCH_BITS == 64
1908	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1909	{
1910	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
1911	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
1912	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
1913	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1914	}
1915	#endif
1916	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1917	}
1918	}
1919
1920
1921	/**
1922	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1923	* lazily while leaving VT-x.
1924	*
1925	* @returns true if it does, false otherwise.
1926	* @param pVCpu The cross context virtual CPU structure.
1927	* @param idMsr The MSR to check.
1928	*/
1929	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t idMsr)
1930	{
1931	NOREF(pVCpu);
1932	#if HC_ARCH_BITS == 64
1933	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1934	{
1935	switch (idMsr)
1936	{
1937	case MSR_K8_LSTAR:
1938	case MSR_K6_STAR:
1939	case MSR_K8_SF_MASK:
1940	case MSR_K8_KERNEL_GS_BASE:
1941	return true;
1942	}
1943	}
1944	#else
1945	RT_NOREF(pVCpu, idMsr);
1946	#endif
1947	return false;
1948	}
1949
1950
1951	/**
1952	* Loads a set of guests MSRs to allow read/passthru to the guest.
1953	*
1954	* The name of this function is slightly confusing. This function does NOT
1955	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1956	* common prefix for functions dealing with "lazy restoration" of the shared
1957	* MSRs.
1958	*
1959	* @param pVCpu The cross context virtual CPU structure.
1960	*
1961	* @remarks No-long-jump zone!!!
1962	*/
1963	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1964	{
1965	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1966	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1967
1968	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1969	#if HC_ARCH_BITS == 64
1970	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1971	{
1972	/*
1973	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1974	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1975	* we can skip a few MSR writes.
1976	*
1977	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1978	* guest MSR values in the guest-CPU context might be different to what's currently
1979	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1980	* CPU, see @bugref{8728}.
1981	*/
1982	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1983	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1984	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
1985	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
1986	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
1987	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
1988	{
1989	#ifdef VBOX_STRICT
1990	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1991	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1992	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1993	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1994	#endif
1995	}
1996	else
1997	{
1998	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1999	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2000	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2001	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2002	}
2003	}
2004	#endif
2005	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2006	}
2007
2008
2009	/**
2010	* Performs lazy restoration of the set of host MSRs if they were previously
2011	* loaded with guest MSR values.
2012	*
2013	* @param pVCpu The cross context virtual CPU structure.
2014	*
2015	* @remarks No-long-jump zone!!!
2016	* @remarks The guest MSRs should have been saved back into the guest-CPU
2017	* context by hmR0VmxImportGuestState()!!!
2018	*/
2019	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2020	{
2021	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2022	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2023
2024	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2025	{
2026	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2027	#if HC_ARCH_BITS == 64
2028	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2029	{
2030	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2031	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2032	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2033	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2034	}
2035	#endif
2036	}
2037	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2038	}
2039
2040
2041	/**
2042	* Verifies that our cached values of the VMCS fields are all consistent with
2043	* what's actually present in the VMCS.
2044	*
2045	* @returns VBox status code.
2046	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2047	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2048	* VMCS content. HMCPU error-field is
2049	* updated, see VMX_VCI_XXX.
2050	* @param pVCpu The cross context virtual CPU structure.
2051	* @param pVmcsInfo The VMCS info. object.
2052	*/
2053	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2054	{
2055	uint32_t u32Val;
2056	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2057	AssertRCReturn(rc, rc);
2058	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2059	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32EntryCtls, u32Val),
2060	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2061	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2062
2063	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2064	AssertRCReturn(rc, rc);
2065	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2066	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ExitCtls, u32Val),
2067	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2068	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2069
2070	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2071	AssertRCReturn(rc, rc);
2072	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2073	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32PinCtls, u32Val),
2074	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2075	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2076
2077	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2078	AssertRCReturn(rc, rc);
2079	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2080	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls, u32Val),
2081	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2082	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2083
2084	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2085	{
2086	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2087	AssertRCReturn(rc, rc);
2088	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2089	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls2, u32Val),
2090	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2091	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2092	}
2093
2094	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2095	AssertRCReturn(rc, rc);
2096	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2097	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32XcptBitmap, u32Val),
2098	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2099	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2100
2101	uint64_t u64Val;
2102	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2103	AssertRCReturn(rc, rc);
2104	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2105	("Cache=%#RX64 VMCS=%#RX64\n", pVmcsInfo->u64TscOffset, u64Val),
2106	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2107	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2108
2109	return VINF_SUCCESS;
2110	}
2111
2112
2113	#ifdef VBOX_STRICT
2114	/**
2115	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2116	*
2117	* @param pVCpu The cross context virtual CPU structure.
2118	* @param pVmcsInfo The VMCS info. object.
2119	*/
2120	static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2121	{
2122	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2123
2124	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2125	{
2126	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2127	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2128	uint64_t uVmcsEferMsrVmcs;
2129	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2130	AssertRC(rc);
2131
2132	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2133	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2134	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2135	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2136	}
2137	}
2138
2139
2140	/**
2141	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2142	* VMCS are correct.
2143	*
2144	* @param pVCpu The cross context virtual CPU structure.
2145	* @param pVmcsInfo The VMCS info. object.
2146	*/
2147	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2148	{
2149	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2150
2151	/* Verify MSR counts in the VMCS are what we think it should be. */
2152	uint32_t cMsrs;
2153	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs);
2154	AssertRC(rc);
2155	Assert(cMsrs == pVmcsInfo->cEntryMsrLoad);
2156
2157	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs);
2158	AssertRC(rc);
2159	Assert(cMsrs == pVmcsInfo->cExitMsrStore);
2160
2161	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs);
2162	AssertRC(rc);
2163	Assert(cMsrs == pVmcsInfo->cExitMsrLoad);
2164
2165	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2166	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2167
2168	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2169	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2170	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2171	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2172	for (uint32_t i = 0; i < cMsrs; i++)
2173	{
2174	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2175	if (fSeparateExitMsrStorePage)
2176	{
2177	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2178	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2179	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2180	}
2181
2182	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2183	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2184	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2185
2186	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2187	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2188	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2189	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2190
2191	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2192	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2193	{
2194	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2195	if (pGuestMsrLoad->u32Msr == MSR_K6_EFER)
2196	{
2197	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2198	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2199	}
2200	else
2201	{
2202	AssertMsgReturnVoid((fMsrpm & (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR))
2203	== (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR),
2204	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2205	}
2206	}
2207
2208	/* Move to the next MSR. */
2209	pHostMsrLoad++;
2210	pGuestMsrLoad++;
2211	pGuestMsrStore++;
2212	}
2213	}
2214	#endif /* VBOX_STRICT */
2215
2216
2217	/**
2218	* Flushes the TLB using EPT.
2219	*
2220	* @returns VBox status code.
2221	* @param pVCpu The cross context virtual CPU structure of the calling
2222	* EMT. Can be NULL depending on @a enmTlbFlush.
2223	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2224	* enmTlbFlush.
2225	* @param enmTlbFlush Type of flush.
2226	*
2227	* @remarks Caller is responsible for making sure this function is called only
2228	* when NestedPaging is supported and providing @a enmTlbFlush that is
2229	* supported by the CPU.
2230	* @remarks Can be called with interrupts disabled.
2231	*/
2232	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2233	{
2234	uint64_t au64Descriptor[2];
2235	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2236	au64Descriptor[0] = 0;
2237	else
2238	{
2239	Assert(pVCpu);
2240	Assert(pVmcsInfo);
2241	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2242	}
2243	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2244
2245	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2246	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2247
2248	if ( RT_SUCCESS(rc)
2249	&& pVCpu)
2250	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2251	}
2252
2253
2254	/**
2255	* Flushes the TLB using VPID.
2256	*
2257	* @returns VBox status code.
2258	* @param pVCpu The cross context virtual CPU structure of the calling
2259	* EMT. Can be NULL depending on @a enmTlbFlush.
2260	* @param enmTlbFlush Type of flush.
2261	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2262	* on @a enmTlbFlush).
2263	*
2264	* @remarks Can be called with interrupts disabled.
2265	*/
2266	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2267	{
2268	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2269
2270	uint64_t au64Descriptor[2];
2271	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2272	{
2273	au64Descriptor[0] = 0;
2274	au64Descriptor[1] = 0;
2275	}
2276	else
2277	{
2278	AssertPtr(pVCpu);
2279	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2280	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2281	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2282	au64Descriptor[1] = GCPtr;
2283	}
2284
2285	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2286	AssertMsg(rc == VINF_SUCCESS,
2287	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2288
2289	if ( RT_SUCCESS(rc)
2290	&& pVCpu)
2291	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2292	NOREF(rc);
2293	}
2294
2295
2296	/**
2297	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2298	* otherwise there is nothing really to invalidate.
2299	*
2300	* @returns VBox status code.
2301	* @param pVCpu The cross context virtual CPU structure.
2302	* @param GCVirt Guest virtual address of the page to invalidate.
2303	*/
2304	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2305	{
2306	AssertPtr(pVCpu);
2307	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2308
2309	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2310	{
2311	/*
2312	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2313	* the EPT case. See @bugref{6043} and @bugref{6177}.
2314	*
2315	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2316	* as this function maybe called in a loop with individual addresses.
2317	*/
2318	PVM pVM = pVCpu->CTX_SUFF(pVM);
2319	if (pVM->hm.s.vmx.fVpid)
2320	{
2321	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2322
2323	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2324	/*
2325	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2326	* where executing INVVPID outside 64-bit mode does not flush translations of
2327	* 64-bit linear addresses, see @bugref{6208#c72}.
2328	*/
2329	if (RT_HI_U32(GCVirt))
2330	fVpidFlush = false;
2331	#endif
2332
2333	if (fVpidFlush)
2334	{
2335	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2336	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2337	}
2338	else
2339	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2340	}
2341	else if (pVM->hm.s.fNestedPaging)
2342	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2343	}
2344
2345	return VINF_SUCCESS;
2346	}
2347
2348
2349	/**
2350	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2351	* case where neither EPT nor VPID is supported by the CPU.
2352	*
2353	* @param pHostCpu The HM physical-CPU structure.
2354	* @param pVCpu The cross context virtual CPU structure.
2355	*
2356	* @remarks Called with interrupts disabled.
2357	*/
2358	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2359	{
2360	AssertPtr(pVCpu);
2361	AssertPtr(pHostCpu);
2362
2363	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2364
2365	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2366	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2367	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2368	pVCpu->hm.s.fForceTLBFlush = false;
2369	return;
2370	}
2371
2372
2373	/**
2374	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2375	*
2376	* @param pHostCpu The HM physical-CPU structure.
2377	* @param pVCpu The cross context virtual CPU structure.
2378	* @param pVmcsInfo The VMCS info. object.
2379	*
2380	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2381	* nomenclature. The reason is, to avoid confusion in compare statements
2382	* since the host-CPU copies are named "ASID".
2383	*
2384	* @remarks Called with interrupts disabled.
2385	*/
2386	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2387	{
2388	#ifdef VBOX_WITH_STATISTICS
2389	bool fTlbFlushed = false;
2390	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2391	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2392	if (!fTlbFlushed) \
2393	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2394	} while (0)
2395	#else
2396	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2397	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2398	#endif
2399
2400	AssertPtr(pVCpu);
2401	AssertPtr(pHostCpu);
2402	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2403
2404	PVM pVM = pVCpu->CTX_SUFF(pVM);
2405	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2406	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2407	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2408
2409	/*
2410	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2411	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2412	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2413	* cannot reuse the current ASID anymore.
2414	*/
2415	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2416	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2417	{
2418	++pHostCpu->uCurrentAsid;
2419	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2420	{
2421	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2422	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2423	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2424	}
2425
2426	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2427	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2428	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2429
2430	/*
2431	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2432	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2433	*/
2434	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2435	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2436	HMVMX_SET_TAGGED_TLB_FLUSHED();
2437	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2438	}
2439	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2440	{
2441	/*
2442	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2443	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2444	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2445	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2446	* mappings, see @bugref{6568}.
2447	*
2448	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2449	*/
2450	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2451	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2452	HMVMX_SET_TAGGED_TLB_FLUSHED();
2453	}
2454
2455	pVCpu->hm.s.fForceTLBFlush = false;
2456	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2457
2458	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
2459	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
2460	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2461	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2462	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2463	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2464	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2465	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2466	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2467
2468	/* Update VMCS with the VPID. */
2469	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2470	AssertRC(rc);
2471
2472	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2473	}
2474
2475
2476	/**
2477	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2478	*
2479	* @param pHostCpu The HM physical-CPU structure.
2480	* @param pVCpu The cross context virtual CPU structure.
2481	* @param pVmcsInfo The VMCS info. object.
2482	*
2483	* @remarks Called with interrupts disabled.
2484	*/
2485	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2486	{
2487	AssertPtr(pVCpu);
2488	AssertPtr(pHostCpu);
2489	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2490	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2491	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2492
2493	/*
2494	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2495	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2496	*/
2497	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2498	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2499	{
2500	pVCpu->hm.s.fForceTLBFlush = true;
2501	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2502	}
2503
2504	/* Check for explicit TLB flushes. */
2505	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2506	{
2507	pVCpu->hm.s.fForceTLBFlush = true;
2508	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2509	}
2510
2511	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2512	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2513
2514	if (pVCpu->hm.s.fForceTLBFlush)
2515	{
2516	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2517	pVCpu->hm.s.fForceTLBFlush = false;
2518	}
2519	}
2520
2521
2522	/**
2523	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2524	*
2525	* @param pHostCpu The HM physical-CPU structure.
2526	* @param pVCpu The cross context virtual CPU structure.
2527	*
2528	* @remarks Called with interrupts disabled.
2529	*/
2530	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2531	{
2532	AssertPtr(pVCpu);
2533	AssertPtr(pHostCpu);
2534	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2535	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2536	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2537
2538	/*
2539	* Force a TLB flush for the first world switch if the current CPU differs from the one we
2540	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2541	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2542	* cannot reuse the current ASID anymore.
2543	*/
2544	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2545	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2546	{
2547	pVCpu->hm.s.fForceTLBFlush = true;
2548	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2549	}
2550
2551	/* Check for explicit TLB flushes. */
2552	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2553	{
2554	/*
2555	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2556	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2557	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
2558	* include fExplicitFlush's too) - an obscure corner case.
2559	*/
2560	pVCpu->hm.s.fForceTLBFlush = true;
2561	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2562	}
2563
2564	PVM pVM = pVCpu->CTX_SUFF(pVM);
2565	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2566	if (pVCpu->hm.s.fForceTLBFlush)
2567	{
2568	++pHostCpu->uCurrentAsid;
2569	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2570	{
2571	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2572	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2573	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2574	}
2575
2576	pVCpu->hm.s.fForceTLBFlush = false;
2577	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2578	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2579	if (pHostCpu->fFlushAsidBeforeUse)
2580	{
2581	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2582	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2583	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2584	{
2585	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2586	pHostCpu->fFlushAsidBeforeUse = false;
2587	}
2588	else
2589	{
2590	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2591	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2592	}
2593	}
2594	}
2595
2596	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2597	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2598	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2599	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2600	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2601	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2602	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2603
2604	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2605	AssertRC(rc);
2606	}
2607
2608
2609	/**
2610	* Flushes the guest TLB entry based on CPU capabilities.
2611	*
2612	* @param pHostCpu The HM physical-CPU structure.
2613	* @param pVCpu The cross context virtual CPU structure.
2614	* @param pVmcsInfo The VMCS info. object.
2615	*
2616	* @remarks Called with interrupts disabled.
2617	*/
2618	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2619	{
2620	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2621	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2622	#endif
2623	PVM pVM = pVCpu->CTX_SUFF(pVM);
2624	switch (pVM->hm.s.vmx.enmTlbFlushType)
2625	{
2626	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
2627	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
2628	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
2629	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
2630	default:
2631	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2632	break;
2633	}
2634	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2635	}
2636
2637
2638	/**
2639	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2640	* TLB entries from the host TLB before VM-entry.
2641	*
2642	* @returns VBox status code.
2643	* @param pVM The cross context VM structure.
2644	*/
2645	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2646	{
2647	/*
2648	* Determine optimal flush type for nested paging.
2649	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
2650	* unrestricted guest execution (see hmR3InitFinalizeR0()).
2651	*/
2652	if (pVM->hm.s.fNestedPaging)
2653	{
2654	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2655	{
2656	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2657	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2658	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2659	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2660	else
2661	{
2662	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2663	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2664	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2665	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2666	}
2667
2668	/* Make sure the write-back cacheable memory type for EPT is supported. */
2669	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2670	{
2671	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2672	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2673	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2674	}
2675
2676	/* EPT requires a page-walk length of 4. */
2677	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2678	{
2679	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2680	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2681	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2682	}
2683	}
2684	else
2685	{
2686	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2687	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2688	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2689	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2690	}
2691	}
2692
2693	/*
2694	* Determine optimal flush type for VPID.
2695	*/
2696	if (pVM->hm.s.vmx.fVpid)
2697	{
2698	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2699	{
2700	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2701	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2702	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2703	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2704	else
2705	{
2706	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2707	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2708	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2709	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2710	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2711	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2712	pVM->hm.s.vmx.fVpid = false;
2713	}
2714	}
2715	else
2716	{
2717	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2718	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2719	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2720	pVM->hm.s.vmx.fVpid = false;
2721	}
2722	}
2723
2724	/*
2725	* Setup the handler for flushing tagged-TLBs.
2726	*/
2727	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2728	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2729	else if (pVM->hm.s.fNestedPaging)
2730	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2731	else if (pVM->hm.s.vmx.fVpid)
2732	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2733	else
2734	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2735	return VINF_SUCCESS;
2736	}
2737
2738
2739	/**
2740	* Sets up the virtual-APIC page address for the VMCS.
2741	*
2742	* @returns VBox status code.
2743	* @param pVmcsInfo The VMCS info. object.
2744	*/
2745	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PCVMXVMCSINFO pVmcsInfo)
2746	{
2747	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
2748	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
2749	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2750	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
2751	}
2752
2753
2754	/**
2755	* Sets up the MSR-bitmap address for the VMCS.
2756	*
2757	* @returns VBox status code.
2758	* @param pVmcsInfo The VMCS info. object.
2759	*/
2760	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PCVMXVMCSINFO pVmcsInfo)
2761	{
2762	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
2763	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
2764	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2765	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
2766	}
2767
2768
2769	/**
2770	* Sets up the APIC-access page address for the VMCS.
2771	*
2772	* @returns VBox status code.
2773	* @param pVCpu The cross context virtual CPU structure.
2774	*/
2775	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
2776	{
2777	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
2778	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
2779	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2780	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
2781	}
2782
2783
2784	/**
2785	* Sets up the VMCS link pointer for the VMCS.
2786	*
2787	* @returns VBox status code.
2788	* @param pVmcsInfo The VMCS info. object.
2789	*/
2790	DECLINLINE(int) hmR0VmxSetupVmcsLinkPtr(PVMXVMCSINFO pVmcsInfo)
2791	{
2792	uint64_t const u64VmcsLinkPtr = pVmcsInfo->u64VmcsLinkPtr;
2793	Assert(u64VmcsLinkPtr == UINT64_C(0xffffffffffffffff)); /* Bits 63:0 MB1. */
2794	return VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, u64VmcsLinkPtr);
2795	}
2796
2797
2798	/**
2799	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
2800	* in the VMCS.
2801	*
2802	* @returns VBox status code.
2803	* @param pVmcsInfo The VMCS info. object.
2804	*/
2805	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMXVMCSINFO pVmcsInfo)
2806	{
2807	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
2808	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
2809	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2810
2811	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
2812	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
2813	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
2814
2815	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
2816	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
2817	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2818
2819	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
2820	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
2821	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
2822	AssertRCReturn(rc, rc);
2823	return VINF_SUCCESS;
2824	}
2825
2826
2827	/**
2828	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
2829	*
2830	* @param pVCpu The cross context virtual CPU structure.
2831	* @param pVmcsInfo The VMCS info. object.
2832	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2833	*/
2834	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2835	{
2836	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
2837
2838	/*
2839	* The guest can access the following MSRs (read, write) without causing
2840	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
2841	*/
2842	PVM pVM = pVCpu->CTX_SUFF(pVM);
2843	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2844	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2845	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2846	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2847	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2848
2849	#ifdef VBOX_STRICT
2850	/** @todo NSTVMX: Remove this later. */
2851	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_IA32_SYSENTER_CS);
2852	Assert((fMsrpm & (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR)) == (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR));
2853
2854	fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K8_GS_BASE);
2855	Assert((fMsrpm & (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR)) == (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR));
2856	#endif
2857
2858	/*
2859	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
2860	* associated with then. We never need to intercept access (writes need to be
2861	* executed without causing a VM-exit, reads will #GP fault anyway).
2862	*
2863	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
2864	* read/write them. We swap the the guest/host MSR value using the
2865	* auto-load/store MSR area.
2866	*/
2867	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2868	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2869	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
2870	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2871	if (pVM->cpum.ro.GuestFeatures.fIbrs)
2872	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2873
2874	/*
2875	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
2876	*/
2877
2878	#if HC_ARCH_BITS == 64
2879	/*
2880	* Allow full read/write access for the following MSRs (mandatory for VT-x)
2881	* required for 64-bit guests.
2882	*/
2883	if (pVM->hm.s.fAllow64BitGuests)
2884	{
2885	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2886	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K6_STAR, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2887	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2888	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2889
2890	# ifdef VBOX_STRICT
2891	fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K8_GS_BASE);
2892	Assert((fMsrpm & (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR)) == (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR));
2893	# endif
2894	}
2895	#endif
2896	}
2897
2898
2899	/**
2900	* Sets up pin-based VM-execution controls in the VMCS.
2901	*
2902	* @returns VBox status code.
2903	* @param pVCpu The cross context virtual CPU structure.
2904	* @param pVmcsInfo The VMCS info. object.
2905	*/
2906	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2907	{
2908	PVM pVM = pVCpu->CTX_SUFF(pVM);
2909	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
2910	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2911
2912	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2913	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2914
2915	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2916	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2917
2918	/* Enable the VMX-preemption timer. */
2919	if (pVM->hm.s.vmx.fUsePreemptTimer)
2920	{
2921	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2922	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
2923	}
2924
2925	#if 0
2926	/* Enable posted-interrupt processing. */
2927	if (pVM->hm.s.fPostedIntrs)
2928	{
2929	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2930	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2931	fVal \|= VMX_PIN_CTL_POSTED_INT;
2932	}
2933	#endif
2934
2935	if ((fVal & fZap) != fVal)
2936	{
2937	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2938	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
2939	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2940	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2941	}
2942
2943	/* Commit it to the VMCS and update our cache. */
2944	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2945	AssertRCReturn(rc, rc);
2946	pVmcsInfo->u32PinCtls = fVal;
2947
2948	return VINF_SUCCESS;
2949	}
2950
2951
2952	/**
2953	* Sets up secondary processor-based VM-execution controls in the VMCS.
2954	*
2955	* @returns VBox status code.
2956	* @param pVCpu The cross context virtual CPU structure.
2957	* @param pVmcsInfo The VMCS info. object.
2958	*/
2959	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2960	{
2961	PVM pVM = pVCpu->CTX_SUFF(pVM);
2962	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
2963	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2964
2965	/* WBINVD causes a VM-exit. */
2966	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2967	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
2968
2969	/* Enable EPT (aka nested-paging). */
2970	if (pVM->hm.s.fNestedPaging)
2971	fVal \|= VMX_PROC_CTLS2_EPT;
2972
2973	/* Enable the INVPCID instruction if supported by the hardware and we expose
2974	it to the guest. Without this, guest executing INVPCID would cause a #UD. */
2975	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2976	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
2977	fVal \|= VMX_PROC_CTLS2_INVPCID;
2978
2979	/* Enable VPID. */
2980	if (pVM->hm.s.vmx.fVpid)
2981	fVal \|= VMX_PROC_CTLS2_VPID;
2982
2983	/* Enable unrestricted guest execution. */
2984	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2985	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
2986
2987	#if 0
2988	if (pVM->hm.s.fVirtApicRegs)
2989	{
2990	/* Enable APIC-register virtualization. */
2991	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
2992	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
2993
2994	/* Enable virtual-interrupt delivery. */
2995	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
2996	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
2997	}
2998	#endif
2999
3000	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
3001	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3002	* done dynamically. */
3003	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3004	{
3005	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3006	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3007	AssertRCReturn(rc, rc);
3008	}
3009
3010	/* Enable the RDTSCP instruction if supported by the hardware and we expose
3011	it to the guest. Without this, guest executing RDTSCP would cause a #UD. */
3012	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
3013	&& pVM->cpum.ro.GuestFeatures.fRdTscP)
3014	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3015
3016	/* Enable Pause-Loop exiting. */
3017	if ( pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT
3018	&& pVM->hm.s.vmx.cPleGapTicks
3019	&& pVM->hm.s.vmx.cPleWindowTicks)
3020	{
3021	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3022
3023	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3024	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3025	AssertRCReturn(rc, rc);
3026	}
3027
3028	if ((fVal & fZap) != fVal)
3029	{
3030	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3031	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3032	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3033	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3034	}
3035
3036	/* Commit it to the VMCS and update our cache. */
3037	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3038	AssertRCReturn(rc, rc);
3039	pVmcsInfo->u32ProcCtls2 = fVal;
3040
3041	return VINF_SUCCESS;
3042	}
3043
3044
3045	/**
3046	* Sets up processor-based VM-execution controls in the VMCS.
3047	*
3048	* @returns VBox status code.
3049	* @param pVCpu The cross context virtual CPU structure.
3050	* @param pVmcsInfo The VMCS info. object.
3051	*/
3052	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3053	{
3054	PVM pVM = pVCpu->CTX_SUFF(pVM);
3055
3056	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3057	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3058
3059	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3060	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3061	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3062	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3063	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3064	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3065	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3066
3067	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3068	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3069	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3070	{
3071	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3072	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3073	}
3074
3075	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3076	if (!pVM->hm.s.fNestedPaging)
3077	{
3078	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3079	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3080	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3081	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3082	}
3083
3084	/* Use TPR shadowing if supported by the CPU. */
3085	if ( PDMHasApic(pVM)
3086	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3087	{
3088	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3089	/* CR8 writes cause a VM-exit based on TPR threshold. */
3090	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3091	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3092	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVmcsInfo);
3093	AssertRCReturn(rc, rc);
3094	}
3095	else
3096	{
3097	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3098	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3099	if (pVM->hm.s.fAllow64BitGuests)
3100	{
3101	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3102	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3103	}
3104	}
3105
3106	/* Use MSR-bitmaps if supported by the CPU. */
3107	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3108	{
3109	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3110	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
3111	AssertRCReturn(rc, rc);
3112	}
3113
3114	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3115	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3116	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3117
3118	if ((fVal & fZap) != fVal)
3119	{
3120	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3121	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3122	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3123	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3124	}
3125
3126	/* Commit it to the VMCS and update our cache. */
3127	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3128	AssertRCReturn(rc, rc);
3129	pVmcsInfo->u32ProcCtls = fVal;
3130
3131	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3132	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo, false /* fIsNstGstVmcs */);
3133
3134	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3135	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3136	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3137
3138	/* Sanity check, should not really happen. */
3139	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3140	{ /* likely */ }
3141	else
3142	{
3143	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3144	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3145	}
3146
3147	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3148	return VINF_SUCCESS;
3149	}
3150
3151
3152	/**
3153	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3154	* Processor-based VM-execution) control fields in the VMCS.
3155	*
3156	* @returns VBox status code.
3157	* @param pVCpu The cross context virtual CPU structure.
3158	* @param pVmcsInfo The VMCS info. object.
3159	*/
3160	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3161	{
3162	/* Set the auto-load/store MSR area addresses in the VMCS. */
3163	int rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
3164	if (RT_SUCCESS(rc))
3165	{
3166	/* Set the VMCS link pointer in the VMCS. */
3167	rc = hmR0VmxSetupVmcsLinkPtr(pVmcsInfo);
3168	if (RT_SUCCESS(rc))
3169	{
3170	/* Set the CR0/CR4 guest/host mask. */
3171	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3172	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3173	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3174	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3175	if (RT_SUCCESS(rc))
3176	{
3177	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3178	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3179	return VINF_SUCCESS;
3180	}
3181	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3182	}
3183	else
3184	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3185	}
3186	else
3187	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3188	return rc;
3189	}
3190
3191
3192	/**
3193	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3194	*
3195	* We shall setup those exception intercepts that don't change during the
3196	* lifetime of the VM here. The rest are done dynamically while loading the
3197	* guest state.
3198	*
3199	* @returns VBox status code.
3200	* @param pVCpu The cross context virtual CPU structure.
3201	* @param pVmcsInfo The VMCS info. object.
3202	*/
3203	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3204	{
3205	/*
3206	* The following exceptions are always intercepted:
3207	*
3208	* #AC - To prevent the guest from hanging the CPU.
3209	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3210	* recursive #DBs can cause a CPU hang.
3211	* #PF - To sync our shadow page tables when nested-paging is not used.
3212	*/
3213	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3214	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3215	\| RT_BIT(X86_XCPT_DB)
3216	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3217
3218	/* Commit it to the VMCS. */
3219	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3220	AssertRCReturn(rc, rc);
3221
3222	/* Update our cache of the exception bitmap. */
3223	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3224	return VINF_SUCCESS;
3225	}
3226
3227
3228	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3229	/**
3230	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3231	*
3232	* @returns VBox status code.
3233	* @param pVCpu The cross context virtual CPU structure.
3234	* @param pVmcsInfo The VMCS info. object.
3235	*/
3236	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3237	{
3238	PVM pVM = pVCpu->CTX_SUFF(pVM);
3239	int rc = hmR0VmxSetupVmcsLinkPtr(pVmcsInfo);
3240	if (RT_SUCCESS(rc))
3241	{
3242	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
3243	if (RT_SUCCESS(rc))
3244	{
3245	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3246	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
3247	if (RT_SUCCESS(rc))
3248	{
3249	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3250	rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3251	if (RT_SUCCESS(rc))
3252	return VINF_SUCCESS;
3253
3254	LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3255	}
3256	else
3257	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3258	}
3259	else
3260	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3261	}
3262	else
3263	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3264
3265	return rc;
3266	}
3267	#endif
3268
3269
3270	/**
3271	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3272	* VMX.
3273	*
3274	* @returns VBox status code.
3275	* @param pVCpu The cross context virtual CPU structure.
3276	* @param pVmcsInfo The VMCS info. object.
3277	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3278	*/
3279	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3280	{
3281	Assert(pVmcsInfo);
3282	Assert(pVmcsInfo->pvVmcs);
3283	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3284
3285	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3286	PVM pVM = pVCpu->CTX_SUFF(pVM);
3287	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3288	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3289
3290	LogFlowFunc(("\n"));
3291
3292	/*
3293	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3294	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3295	*/
3296	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3297	if (RT_SUCCESS(rc))
3298	{
3299	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3300	if (RT_SUCCESS(rc))
3301	{
3302	if (!fIsNstGstVmcs)
3303	{
3304	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3305	if (RT_SUCCESS(rc))
3306	{
3307	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
3308	if (RT_SUCCESS(rc))
3309	{
3310	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
3311	if (RT_SUCCESS(rc))
3312	{
3313	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
3314	if (RT_SUCCESS(rc))
3315	{ /* likely */ }
3316	else
3317	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
3318	}
3319	else
3320	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
3321	}
3322	else
3323	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
3324	}
3325	else
3326	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
3327	}
3328	else
3329	{
3330	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3331	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
3332	if (RT_SUCCESS(rc))
3333	{ /* likely */ }
3334	else
3335	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
3336	#else
3337	AssertFailed();
3338	#endif
3339	}
3340	}
3341	else
3342	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
3343	}
3344	else
3345	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
3346
3347	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
3348	if (RT_SUCCESS(rc))
3349	{
3350	rc = hmR0VmxClearVmcs(pVmcsInfo);
3351	if (RT_SUCCESS(rc))
3352	{ /* likely */ }
3353	else
3354	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
3355	}
3356
3357	/*
3358	* Update the last-error record both for failures and success, so we
3359	* can propagate the status code back to ring-3 for diagnostics.
3360	*/
3361	hmR0VmxUpdateErrorRecord(pVCpu, rc);
3362	NOREF(pszVmcs);
3363	return rc;
3364	}
3365
3366
3367	/**
3368	* Does global VT-x initialization (called during module initialization).
3369	*
3370	* @returns VBox status code.
3371	*/
3372	VMMR0DECL(int) VMXR0GlobalInit(void)
3373	{
3374	#ifdef HMVMX_USE_FUNCTION_TABLE
3375	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
3376	# ifdef VBOX_STRICT
3377	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
3378	Assert(g_apfnVMExitHandlers[i]);
3379	# endif
3380	#endif
3381	return VINF_SUCCESS;
3382	}
3383
3384
3385	/**
3386	* Does global VT-x termination (called during module termination).
3387	*/
3388	VMMR0DECL(void) VMXR0GlobalTerm()
3389	{
3390	/* Nothing to do currently. */
3391	}
3392
3393
3394	/**
3395	* Sets up and activates VT-x on the current CPU.
3396	*
3397	* @returns VBox status code.
3398	* @param pHostCpu The HM physical-CPU structure.
3399	* @param pVM The cross context VM structure. Can be
3400	* NULL after a host resume operation.
3401	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
3402	* fEnabledByHost is @c true).
3403	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
3404	* @a fEnabledByHost is @c true).
3405	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
3406	* enable VT-x on the host.
3407	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
3408	*/
3409	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
3410	PCSUPHWVIRTMSRS pHwvirtMsrs)
3411	{
3412	Assert(pHostCpu);
3413	Assert(pHwvirtMsrs);
3414	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3415
3416	/* Enable VT-x if it's not already enabled by the host. */
3417	if (!fEnabledByHost)
3418	{
3419	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
3420	if (RT_FAILURE(rc))
3421	return rc;
3422	}
3423
3424	/*
3425	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
3426	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
3427	* invalidated when flushing by VPID.
3428	*/
3429	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3430	{
3431	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
3432	pHostCpu->fFlushAsidBeforeUse = false;
3433	}
3434	else
3435	pHostCpu->fFlushAsidBeforeUse = true;
3436
3437	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
3438	++pHostCpu->cTlbFlushes;
3439
3440	return VINF_SUCCESS;
3441	}
3442
3443
3444	/**
3445	* Deactivates VT-x on the current CPU.
3446	*
3447	* @returns VBox status code.
3448	* @param pvCpuPage Pointer to the VMXON region.
3449	* @param HCPhysCpuPage Physical address of the VMXON region.
3450	*
3451	* @remarks This function should never be called when SUPR0EnableVTx() or
3452	* similar was used to enable VT-x on the host.
3453	*/
3454	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
3455	{
3456	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
3457
3458	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3459	return hmR0VmxLeaveRootMode();
3460	}
3461
3462
3463	/**
3464	* Does per-VM VT-x initialization.
3465	*
3466	* @returns VBox status code.
3467	* @param pVM The cross context VM structure.
3468	*/
3469	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
3470	{
3471	LogFlowFunc(("pVM=%p\n", pVM));
3472
3473	int rc = hmR0VmxStructsAlloc(pVM);
3474	if (RT_FAILURE(rc))
3475	{
3476	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
3477	return rc;
3478	}
3479
3480	return VINF_SUCCESS;
3481	}
3482
3483
3484	/**
3485	* Does per-VM VT-x termination.
3486	*
3487	* @returns VBox status code.
3488	* @param pVM The cross context VM structure.
3489	*/
3490	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
3491	{
3492	LogFlowFunc(("pVM=%p\n", pVM));
3493
3494	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3495	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
3496	{
3497	Assert(pVM->hm.s.vmx.pvScratch);
3498	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
3499	}
3500	#endif
3501	hmR0VmxStructsFree(pVM);
3502	return VINF_SUCCESS;
3503	}
3504
3505
3506	/**
3507	* Sets up the VM for execution using hardware-assisted VMX.
3508	* This function is only called once per-VM during initialization.
3509	*
3510	* @returns VBox status code.
3511	* @param pVM The cross context VM structure.
3512	*/
3513	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
3514	{
3515	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
3516	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3517
3518	LogFlowFunc(("pVM=%p\n", pVM));
3519
3520	/*
3521	* At least verify if VMX is enabled, since we can't check if we're in
3522	* VMX root mode or not without causing a #GP.
3523	*/
3524	RTCCUINTREG const uHostCR4 = ASMGetCR4();
3525	if (RT_LIKELY(uHostCR4 & X86_CR4_VMXE))
3526	{ /* likely */ }
3527	else
3528	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
3529
3530	/*
3531	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
3532	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
3533	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
3534	*/
3535	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3536	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
3537	\|\| !pVM->hm.s.vmx.pRealModeTSS))
3538	{
3539	LogRelFunc(("Invalid real-on-v86 state.\n"));
3540	return VERR_INTERNAL_ERROR;
3541	}
3542
3543	/* Initialize these always, see hmR3InitFinalizeR0().*/
3544	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
3545	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
3546
3547	/* Setup the tagged-TLB flush handlers. */
3548	int rc = hmR0VmxSetupTaggedTlb(pVM);
3549	if (RT_FAILURE(rc))
3550	{
3551	LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
3552	return rc;
3553	}
3554
3555	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
3556	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
3557	#if HC_ARCH_BITS == 64
3558	if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
3559	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
3560	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
3561	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
3562	#endif
3563
3564	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3565	{
3566	PVMCPU pVCpu = &pVM->aCpus[idCpu];
3567	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
3568
3569	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
3570	if (RT_SUCCESS(rc))
3571	{
3572	#if HC_ARCH_BITS == 32
3573	hmR0VmxInitVmcsReadCache(pVCpu);
3574	#endif
3575	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3576	if (pVM->cpum.ro.GuestFeatures.fVmx)
3577	{
3578	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
3579	if (RT_SUCCESS(rc))
3580	{ /* likely */ }
3581	else
3582	{
3583	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
3584	return rc;
3585	}
3586	}
3587	#endif
3588	}
3589	else
3590	{
3591	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
3592	return rc;
3593	}
3594	}
3595
3596	return VINF_SUCCESS;
3597	}
3598
3599
3600	#if HC_ARCH_BITS == 32
3601	# ifdef VBOX_ENABLE_64_BITS_GUESTS
3602	/**
3603	* Check if guest state allows safe use of 32-bit switcher again.
3604	*
3605	* Segment bases and protected mode structures must be 32-bit addressable
3606	* because the 32-bit switcher will ignore high dword when writing these VMCS
3607	* fields. See @bugref{8432} for details.
3608	*
3609	* @returns true if safe, false if must continue to use the 64-bit switcher.
3610	* @param pCtx Pointer to the guest-CPU context.
3611	*
3612	* @remarks No-long-jump zone!!!
3613	*/
3614	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
3615	{
3616	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
3617	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
3618	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3619	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3620	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
3621	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3622	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
3623	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
3624	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3625	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3626
3627	/* All good, bases are 32-bit. */
3628	return true;
3629	}
3630	# endif
3631
3632	/**
3633	* Executes the specified handler in 64-bit mode.
3634	*
3635	* @returns VBox status code (no informational status codes).
3636	* @param pVCpu The cross context virtual CPU structure.
3637	* @param enmOp The operation to perform.
3638	* @param cParams Number of parameters.
3639	* @param paParam Array of 32-bit parameters.
3640	*/
3641	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
3642	{
3643	PVM pVM = pVCpu->CTX_SUFF(pVM);
3644	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
3645	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
3646	Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
3647	Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
3648
3649	#ifdef VBOX_STRICT
3650	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
3651	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
3652
3653	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
3654	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
3655	#endif
3656
3657	/* Disable interrupts. */
3658	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
3659
3660	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
3661	RTCPUID idHostCpu = RTMpCpuId();
3662	CPUMR0SetLApic(pVCpu, idHostCpu);
3663	#endif
3664
3665	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
3666
3667	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3668	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3669
3670	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
3671	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3672	hmR0VmxClearVmcs(pVmcsInfo);
3673
3674	/* Leave VMX root mode and disable VMX. */
3675	VMXDisable();
3676	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3677
3678	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
3679	CPUMSetHyperEIP(pVCpu, enmOp);
3680	for (int i = (int)cParams - 1; i >= 0; i--)
3681	CPUMPushHyper(pVCpu, paParam[i]);
3682
3683	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
3684
3685	/* Call the switcher. */
3686	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
3687	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
3688
3689	/* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
3690	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
3691
3692	/* Re-enter VMX root mode. */
3693	int rc2 = VMXEnable(HCPhysCpuPage);
3694	if (RT_FAILURE(rc2))
3695	{
3696	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3697	ASMSetFlags(fOldEFlags);
3698	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
3699	return rc2;
3700	}
3701
3702	/* Restore the VMCS as the current VMCS. */
3703	rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
3704	AssertRC(rc2);
3705	Assert(!(ASMGetFlags() & X86_EFL_IF));
3706	ASMSetFlags(fOldEFlags);
3707	return rc;
3708	}
3709
3710
3711	/**
3712	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
3713	* supporting 64-bit guests.
3714	*
3715	* @returns VBox status code.
3716	* @param fResume Whether to VMLAUNCH or VMRESUME.
3717	* @param pCtx Pointer to the guest-CPU context.
3718	* @param pCache Pointer to the VMCS batch cache.
3719	* @param pVM The cross context VM structure.
3720	* @param pVCpu The cross context virtual CPU structure.
3721	*/
3722	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
3723	{
3724	NOREF(fResume);
3725
3726	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3727	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3728
3729	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3730	pCache->uPos = 1;
3731	pCache->interPD = PGMGetInterPaeCR3(pVM);
3732	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
3733	#endif
3734
3735	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3736	pCache->TestIn.HCPhysCpuPage = 0;
3737	pCache->TestIn.HCPhysVmcs = 0;
3738	pCache->TestIn.pCache = 0;
3739	pCache->TestOut.HCPhysVmcs = 0;
3740	pCache->TestOut.pCache = 0;
3741	pCache->TestOut.pCtx = 0;
3742	pCache->TestOut.eflags = 0;
3743	#else
3744	NOREF(pCache);
3745	#endif
3746
3747	uint32_t aParam[10];
3748	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
3749	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
3750	aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
3751	aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
3752	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
3753	aParam[5] = 0;
3754	aParam[6] = VM_RC_ADDR(pVM, pVM);
3755	aParam[7] = 0;
3756	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
3757	aParam[9] = 0;
3758
3759	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3760	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
3761	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
3762	#endif
3763	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
3764
3765	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3766	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
3767	Assert(pCtx->dr[4] == 10);
3768	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
3769	#endif
3770
3771	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3772	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
3773	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3774	pVCpu->hm.s.vmx.HCPhysVmcs));
3775	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3776	pCache->TestOut.HCPhysVmcs));
3777	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
3778	pCache->TestOut.pCache));
3779	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
3780	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
3781	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
3782	pCache->TestOut.pCtx));
3783	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
3784	#endif
3785	NOREF(pCtx);
3786	return rc;
3787	}
3788	#endif
3789
3790
3791	/**
3792	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
3793	* the VMCS.
3794	*
3795	* @returns VBox status code.
3796	*/
3797	static int hmR0VmxExportHostControlRegs(void)
3798	{
3799	RTCCUINTREG uReg = ASMGetCR0();
3800	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
3801	AssertRCReturn(rc, rc);
3802
3803	uReg = ASMGetCR3();
3804	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
3805	AssertRCReturn(rc, rc);
3806
3807	uReg = ASMGetCR4();
3808	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
3809	AssertRCReturn(rc, rc);
3810	return rc;
3811	}
3812
3813
3814	/**
3815	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
3816	* the host-state area in the VMCS.
3817	*
3818	* @returns VBox status code.
3819	* @param pVCpu The cross context virtual CPU structure.
3820	*/
3821	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
3822	{
3823	#if HC_ARCH_BITS == 64
3824	/**
3825	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
3826	* requirements. See hmR0VmxExportHostSegmentRegs().
3827	*/
3828	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
3829	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
3830	{ \
3831	bool fValidSelector = true; \
3832	if ((selValue) & X86_SEL_LDT) \
3833	{ \
3834	uint32_t uAttr = ASMGetSegAttr((selValue)); \
3835	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
3836	} \
3837	if (fValidSelector) \
3838	{ \
3839	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
3840	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
3841	} \
3842	(selValue) = 0; \
3843	}
3844
3845	/*
3846	* If we've executed guest code using hardware-assisted VMX, the host-state bits
3847	* will be messed up. We should -not- save the messed up state without restoring
3848	* the original host-state, see @bugref{7240}.
3849	*
3850	* This apparently can happen (most likely the FPU changes), deal with it rather than
3851	* asserting. Was observed booting Solaris 10u10 32-bit guest.
3852	*/
3853	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
3854	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
3855	{
3856	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
3857	pVCpu->idCpu));
3858	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
3859	}
3860	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
3861	#else
3862	RT_NOREF(pVCpu);
3863	#endif
3864
3865	/*
3866	* Host DS, ES, FS and GS segment registers.
3867	*/
3868	#if HC_ARCH_BITS == 64
3869	RTSEL uSelDS = ASMGetDS();
3870	RTSEL uSelES = ASMGetES();
3871	RTSEL uSelFS = ASMGetFS();
3872	RTSEL uSelGS = ASMGetGS();
3873	#else
3874	RTSEL uSelDS = 0;
3875	RTSEL uSelES = 0;
3876	RTSEL uSelFS = 0;
3877	RTSEL uSelGS = 0;
3878	#endif
3879
3880	/*
3881	* Host CS and SS segment registers.
3882	*/
3883	RTSEL uSelCS = ASMGetCS();
3884	RTSEL uSelSS = ASMGetSS();
3885
3886	/*
3887	* Host TR segment register.
3888	*/
3889	RTSEL uSelTR = ASMGetTR();
3890
3891	#if HC_ARCH_BITS == 64
3892	/*
3893	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
3894	* gain VM-entry and restore them before we get preempted.
3895	*
3896	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
3897	*/
3898	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3899	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3900	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3901	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3902	# undef VMXLOCAL_ADJUST_HOST_SEG
3903	#endif
3904
3905	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
3906	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
3907	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
3908	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
3909	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
3910	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
3911	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
3912	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
3913	Assert(uSelCS);
3914	Assert(uSelTR);
3915
3916	/* Write these host selector fields into the host-state area in the VMCS. */
3917	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3918	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3919	#if HC_ARCH_BITS == 64
3920	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3921	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3922	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3923	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3924	#else
3925	NOREF(uSelDS);
3926	NOREF(uSelES);
3927	NOREF(uSelFS);
3928	NOREF(uSelGS);
3929	#endif
3930	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3931	AssertRCReturn(rc, rc);
3932
3933	/*
3934	* Host GDTR and IDTR.
3935	*/
3936	RTGDTR Gdtr;
3937	RTIDTR Idtr;
3938	RT_ZERO(Gdtr);
3939	RT_ZERO(Idtr);
3940	ASMGetGDTR(&Gdtr);
3941	ASMGetIDTR(&Idtr);
3942	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3943	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3944	AssertRCReturn(rc, rc);
3945
3946	#if HC_ARCH_BITS == 64
3947	/*
3948	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
3949	* them to the maximum limit (0xffff) on every VM-exit.
3950	*/
3951	if (Gdtr.cbGdt != 0xffff)
3952	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
3953
3954	/*
3955	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
3956	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
3957	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
3958	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
3959	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
3960	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
3961	* at 0xffff on hosts where we are sure it won't cause trouble.
3962	*/
3963	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
3964	if (Idtr.cbIdt < 0x0fff)
3965	# else
3966	if (Idtr.cbIdt != 0xffff)
3967	# endif
3968	{
3969	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
3970	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3971	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3972	}
3973	#endif
3974
3975	/*
3976	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3977	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3978	* RPL should be too in most cases.
3979	*/
3980	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3981	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3982
3983	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3984	#if HC_ARCH_BITS == 64
3985	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
3986
3987	/*
3988	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3989	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3990	* restoration if the host has something else. Task switching is not supported in 64-bit
3991	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3992	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3993	*
3994	* [1] See Intel spec. 3.5 "System Descriptor Types".
3995	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3996	*/
3997	PVM pVM = pVCpu->CTX_SUFF(pVM);
3998	Assert(pDesc->System.u4Type == 11);
3999	if ( pDesc->System.u16LimitLow != 0x67
4000	\|\| pDesc->System.u4LimitHigh)
4001	{
4002	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4003	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4004	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4005	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4006	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4007	}
4008
4009	/*
4010	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4011	*/
4012	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4013	{
4014	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4015	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4016	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4017	{
4018	/* The GDT is read-only but the writable GDT is available. */
4019	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4020	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4021	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4022	AssertRCReturn(rc, rc);
4023	}
4024	}
4025	#else
4026	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4027	#endif
4028	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4029	AssertRCReturn(rc, rc);
4030
4031	/*
4032	* Host FS base and GS base.
4033	*/
4034	#if HC_ARCH_BITS == 64
4035	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4036	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4037	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4038	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4039	AssertRCReturn(rc, rc);
4040
4041	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4042	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4043	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4044	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4045	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4046	#endif
4047	return VINF_SUCCESS;
4048	}
4049
4050
4051	/**
4052	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4053	* host-state area of the VMCS.
4054	*
4055	* These MSRs will be automatically restored on the host after every successful
4056	* VM-exit.
4057	*
4058	* @returns VBox status code.
4059	* @param pVCpu The cross context virtual CPU structure.
4060	*
4061	* @remarks No-long-jump zone!!!
4062	*/
4063	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4064	{
4065	AssertPtr(pVCpu);
4066
4067	/*
4068	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4069	* rather than swapping them on every VM-entry.
4070	*/
4071	hmR0VmxLazySaveHostMsrs(pVCpu);
4072
4073	/*
4074	* Host Sysenter MSRs.
4075	*/
4076	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4077	#if HC_ARCH_BITS == 32
4078	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4079	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4080	#else
4081	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4082	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4083	#endif
4084	AssertRCReturn(rc, rc);
4085
4086	/*
4087	* Host EFER MSR.
4088	*
4089	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4090	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4091	*/
4092	PVM pVM = pVCpu->CTX_SUFF(pVM);
4093	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4094	{
4095	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4096	AssertRCReturn(rc, rc);
4097	}
4098
4099	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4100	* hmR0VmxExportGuestEntryExitCtls(). */
4101
4102	return VINF_SUCCESS;
4103	}
4104
4105
4106	/**
4107	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4108	*
4109	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4110	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4111	*
4112	* @returns true if we need to load guest EFER, false otherwise.
4113	* @param pVCpu The cross context virtual CPU structure.
4114	*
4115	* @remarks Requires EFER, CR4.
4116	* @remarks No-long-jump zone!!!
4117	*/
4118	static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu)
4119	{
4120	#ifdef HMVMX_ALWAYS_SWAP_EFER
4121	RT_NOREF(pVCpu);
4122	return true;
4123	#else
4124	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4125	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4126	/* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4127	if (CPUMIsGuestInLongModeEx(pCtx))
4128	return false;
4129	#endif
4130
4131	PVM pVM = pVCpu->CTX_SUFF(pVM);
4132	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4133	uint64_t const u64GuestEfer = pCtx->msrEFER;
4134
4135	/*
4136	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4137	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4138	*/
4139	if ( CPUMIsGuestInLongModeEx(pCtx)
4140	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4141	return true;
4142
4143	/*
4144	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4145	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4146	*
4147	* See Intel spec. 4.5 "IA-32e Paging".
4148	* See Intel spec. 4.1.1 "Three Paging Modes".
4149	*
4150	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4151	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4152	*/
4153	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4154	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4155	if ( (pCtx->cr4 & X86_CR4_PAE)
4156	&& (pCtx->cr0 & X86_CR0_PG)
4157	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4158	{
4159	/* Assert that host is NX capable. */
4160	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4161	return true;
4162	}
4163
4164	return false;
4165	#endif
4166	}
4167
4168	/**
4169	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4170	* VMCS.
4171	*
4172	* This is typically required when the guest changes paging mode.
4173	*
4174	* @returns VBox status code.
4175	* @param pVCpu The cross context virtual CPU structure.
4176	* @param pVmxTransient The VMX-transient structure.
4177	*
4178	* @remarks Requires EFER.
4179	* @remarks No-long-jump zone!!!
4180	*/
4181	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4182	{
4183	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4184	{
4185	PVM pVM = pVCpu->CTX_SUFF(pVM);
4186	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4187
4188	/*
4189	* VM-entry controls.
4190	*/
4191	{
4192	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4193	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4194
4195	/*
4196	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4197	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4198	*
4199	* For nested-guests, this is a mandatory VM-entry control. It's also
4200	* required because we do not want to leak host bits to the nested-guest.
4201	*/
4202	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4203
4204	/*
4205	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4206	*
4207	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4208	* required to get the nested-guest working with hardware-assisted VMX execution.
4209	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested-hypervisor
4210	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4211	* here rather than while merging the guest VMCS controls.
4212	*/
4213	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4214	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4215	else
4216	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4217
4218	/*
4219	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4220	*
4221	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4222	* regardless of whether the nested-guest VMCS specifies it because we are free to
4223	* load whatever MSRs we require and we do not need to modify the guest visible copy
4224	* of the VM-entry MSR load area.
4225	*/
4226	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4227	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4228	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4229	else
4230	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4231
4232	/*
4233	* The following should -not- be set (since we're not in SMM mode):
4234	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4235	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4236	*/
4237
4238	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4239	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4240
4241	if ((fVal & fZap) == fVal)
4242	{ /* likely */ }
4243	else
4244	{
4245	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4246	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4247	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4248	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4249	}
4250
4251	/* Commit it to the VMCS. */
4252	if (pVmcsInfo->u32EntryCtls != fVal)
4253	{
4254	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4255	AssertRCReturn(rc, rc);
4256	pVmcsInfo->u32EntryCtls = fVal;
4257	}
4258	}
4259
4260	/*
4261	* VM-exit controls.
4262	*/
4263	{
4264	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4265	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4266
4267	/*
4268	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4269	* supported the 1-setting of this bit.
4270	*
4271	* For nested-guests, we set the "save debug controls" as the converse
4272	* "load debug controls" is mandatory for nested-guests anyway.
4273	*/
4274	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4275
4276	/*
4277	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4278	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4279	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4280	* hmR0VmxExportHostMsrs().
4281	*
4282	* For nested-guests, we always set this bit as we do not support 32-bit
4283	* hosts.
4284	*/
4285	#if HC_ARCH_BITS == 64
4286	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4287	#else
4288	Assert(!pVmxTransient->fIsNestedGuest);
4289	Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
4290	\|\| pVmcsInfo->pfnStartVM == VMXR0StartVM32);
4291	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
4292	if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
4293	{
4294	/* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
4295	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4296	}
4297	else
4298	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
4299	#endif
4300
4301	/*
4302	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4303	*
4304	* For nested-guests, we should use the "save IA32_EFER" control if we also
4305	* used the "load IA32_EFER" control while exporting VM-entry controls.
4306	*/
4307	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4308	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4309	{
4310	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4311	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4312	}
4313
4314	/*
4315	* Enable saving of the VMX-preemption timer value on VM-exit.
4316	* For nested-guests, currently not exposed/used.
4317	*/
4318	if ( pVM->hm.s.vmx.fUsePreemptTimer
4319	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4320	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4321
4322	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4323	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4324
4325	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4326	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4327	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4328
4329	if ((fVal & fZap) == fVal)
4330	{ /* likely */ }
4331	else
4332	{
4333	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4334	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4335	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4336	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4337	}
4338
4339	/* Commit it to the VMCS. */
4340	if (pVmcsInfo->u32ExitCtls != fVal)
4341	{
4342	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4343	AssertRCReturn(rc, rc);
4344	pVmcsInfo->u32ExitCtls = fVal;
4345	}
4346	}
4347
4348	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4349	}
4350	return VINF_SUCCESS;
4351	}
4352
4353
4354	/**
4355	* Sets the TPR threshold in the VMCS.
4356	*
4357	* @returns VBox status code.
4358	* @param pVCpu The cross context virtual CPU structure.
4359	* @param pVmcsInfo The VMCS info. object.
4360	* @param u32TprThreshold The TPR threshold (task-priority class only).
4361	*/
4362	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4363	{
4364	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4365	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4366	RT_NOREF2(pVCpu, pVmcsInfo);
4367	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4368	}
4369
4370
4371	/**
4372	* Exports the guest APIC TPR state into the VMCS.
4373	*
4374	* @returns VBox status code.
4375	* @param pVCpu The cross context virtual CPU structure.
4376	* @param pVmxTransient The VMX-transient structure.
4377	*
4378	* @remarks No-long-jump zone!!!
4379	*/
4380	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4381	{
4382	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4383	{
4384	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4385
4386	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4387	if (!pVmxTransient->fIsNestedGuest)
4388	{
4389	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4390	&& APICIsEnabled(pVCpu))
4391	{
4392	/*
4393	* Setup TPR shadowing.
4394	*/
4395	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4396	{
4397	bool fPendingIntr = false;
4398	uint8_t u8Tpr = 0;
4399	uint8_t u8PendingIntr = 0;
4400	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4401	AssertRCReturn(rc, rc);
4402
4403	/*
4404	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4405	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4406	* priority of the pending interrupt so we can deliver the interrupt. If there
4407	* are no interrupts pending, set threshold to 0 to not cause any
4408	* TPR-below-threshold VM-exits.
4409	*/
4410	Assert(pVmcsInfo->pbVirtApic);
4411	pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
4412	uint32_t u32TprThreshold = 0;
4413	if (fPendingIntr)
4414	{
4415	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4416	(which is the Task-Priority Class). */
4417	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4418	const uint8_t u8TprPriority = u8Tpr >> 4;
4419	if (u8PendingPriority <= u8TprPriority)
4420	u32TprThreshold = u8PendingPriority;
4421	}
4422
4423	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
4424	AssertRCReturn(rc, rc);
4425	}
4426	}
4427	}
4428	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4429	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4430	}
4431	return VINF_SUCCESS;
4432	}
4433
4434
4435	/**
4436	* Gets the guest interruptibility-state.
4437	*
4438	* @returns Guest's interruptibility-state.
4439	* @param pVCpu The cross context virtual CPU structure.
4440	* @param pVmcsInfo The VMCS info. object.
4441	*
4442	* @remarks No-long-jump zone!!!
4443	*/
4444	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
4445	{
4446	/*
4447	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4448	*/
4449	uint32_t fIntrState = 0;
4450	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4451	{
4452	/* If inhibition is active, RIP and RFLAGS should've been updated
4453	(i.e. read previously from the VMCS or from ring-3). */
4454	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4455	#ifdef VBOX_STRICT
4456	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
4457	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4458	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
4459	#endif
4460	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
4461	{
4462	if (pCtx->eflags.Bits.u1IF)
4463	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4464	else
4465	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
4466	}
4467	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4468	{
4469	/*
4470	* We can clear the inhibit force flag as even if we go back to the recompiler
4471	* without executing guest code in VT-x, the flag's condition to be cleared is
4472	* met and thus the cleared state is correct.
4473	*/
4474	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4475	}
4476	}
4477
4478	/*
4479	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
4480	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
4481	* setting this would block host-NMIs and IRET will not clear the blocking.
4482	*
4483	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
4484	*
4485	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
4486	*/
4487	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
4488	&& CPUMIsGuestNmiBlocking(pVCpu))
4489	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
4490
4491	return fIntrState;
4492	}
4493
4494
4495	/**
4496	* Exports the exception intercepts required for guest execution in the VMCS.
4497	*
4498	* @returns VBox status code.
4499	* @param pVCpu The cross context virtual CPU structure.
4500	* @param pVmxTransient The VMX-transient structure.
4501	*
4502	* @remarks No-long-jump zone!!!
4503	*/
4504	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4505	{
4506	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
4507	{
4508	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
4509	if ( !pVmxTransient->fIsNestedGuest
4510	&& pVCpu->hm.s.fGIMTrapXcptUD)
4511	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
4512	else
4513	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
4514
4515	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
4516	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
4517	}
4518	return VINF_SUCCESS;
4519	}
4520
4521
4522	/**
4523	* Exports the guest's RIP into the guest-state area in the VMCS.
4524	*
4525	* @returns VBox status code.
4526	* @param pVCpu The cross context virtual CPU structure.
4527	*
4528	* @remarks No-long-jump zone!!!
4529	*/
4530	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
4531	{
4532	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4533	{
4534	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
4535
4536	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
4537	AssertRCReturn(rc, rc);
4538
4539	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
4540	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
4541	}
4542	return VINF_SUCCESS;
4543	}
4544
4545
4546	/**
4547	* Exports the guest's RSP into the guest-state area in the VMCS.
4548	*
4549	* @returns VBox status code.
4550	* @param pVCpu The cross context virtual CPU structure.
4551	*
4552	* @remarks No-long-jump zone!!!
4553	*/
4554	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
4555	{
4556	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
4557	{
4558	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
4559
4560	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
4561	AssertRCReturn(rc, rc);
4562
4563	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
4564	}
4565	return VINF_SUCCESS;
4566	}
4567
4568
4569	/**
4570	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
4571	*
4572	* @returns VBox status code.
4573	* @param pVCpu The cross context virtual CPU structure.
4574	* @param pVmxTransient The VMX-transient structure.
4575	*
4576	* @remarks No-long-jump zone!!!
4577	*/
4578	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4579	{
4580	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
4581	{
4582	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
4583
4584	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
4585	Let us assert it as such and use 32-bit VMWRITE. */
4586	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
4587	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
4588	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
4589	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
4590
4591	/*
4592	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
4593	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
4594	* can run the real-mode guest code under Virtual 8086 mode.
4595	*/
4596	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4597	if (pVmcsInfo->RealMode.fRealOnV86Active)
4598	{
4599	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4600	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4601	Assert(!pVmxTransient->fIsNestedGuest);
4602	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
4603	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
4604	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
4605	}
4606
4607	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
4608	AssertRCReturn(rc, rc);
4609
4610	/*
4611	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
4612	*
4613	* We must avoid setting any automatic debug exceptions delivery when single-stepping
4614	* through the hypervisor debugger using EFLAGS.TF.
4615	*/
4616	if ( !pVmxTransient->fIsNestedGuest
4617	&& !pVCpu->hm.s.fSingleInstruction
4618	&& fEFlags.Bits.u1TF)
4619	{
4620	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
4621	* premature trips to ring-3 esp since IEM does not yet handle it. */
4622	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
4623	AssertRCReturn(rc, rc);
4624	}
4625	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
4626	* nested-guest VMCS. */
4627
4628	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
4629	Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
4630	}
4631	return VINF_SUCCESS;
4632	}
4633
4634
4635	/**
4636	* Exports the guest CR0 control register into the guest-state area in the VMCS.
4637	*
4638	* The guest FPU state is always pre-loaded hence we don't need to bother about
4639	* sharing FPU related CR0 bits between the guest and host.
4640	*
4641	* @returns VBox status code.
4642	* @param pVCpu The cross context virtual CPU structure.
4643	* @param pVmxTransient The VMX-transient structure.
4644	*
4645	* @remarks No-long-jump zone!!!
4646	*/
4647	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4648	{
4649	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
4650	{
4651	PVM pVM = pVCpu->CTX_SUFF(pVM);
4652	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4653
4654	/*
4655	* Figure out fixed CR0 bits in VMX operation.
4656	*/
4657	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4658	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4659	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4660	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
4661	else
4662	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
4663
4664	if (!pVmxTransient->fIsNestedGuest)
4665	{
4666	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4667	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4668	uint64_t const u64ShadowCr0 = u64GuestCr0;
4669	Assert(!RT_HI_U32(u64GuestCr0));
4670
4671	/*
4672	* Setup VT-x's view of the guest CR0.
4673	*/
4674	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
4675	if (pVM->hm.s.fNestedPaging)
4676	{
4677	if (CPUMIsGuestPagingEnabled(pVCpu))
4678	{
4679	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
4680	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
4681	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
4682	}
4683	else
4684	{
4685	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
4686	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
4687	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
4688	}
4689
4690	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
4691	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4692	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
4693	}
4694	else
4695	{
4696	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
4697	u64GuestCr0 \|= X86_CR0_WP;
4698	}
4699
4700	/*
4701	* Guest FPU bits.
4702	*
4703	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
4704	* using CR0.TS.
4705	*
4706	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
4707	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
4708	*/
4709	u64GuestCr0 \|= X86_CR0_NE;
4710
4711	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
4712	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
4713
4714	/*
4715	* Update exception intercepts.
4716	*/
4717	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
4718	if (pVmcsInfo->RealMode.fRealOnV86Active)
4719	{
4720	Assert(PDMVmmDevHeapIsEnabled(pVM));
4721	Assert(pVM->hm.s.vmx.pRealModeTSS);
4722	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
4723	}
4724	else
4725	{
4726	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
4727	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
4728	if (fInterceptMF)
4729	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
4730	}
4731
4732	/* Additional intercepts for debugging, define these yourself explicitly. */
4733	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4734	uXcptBitmap \|= 0
4735	\| RT_BIT(X86_XCPT_BP)
4736	\| RT_BIT(X86_XCPT_DE)
4737	\| RT_BIT(X86_XCPT_NM)
4738	\| RT_BIT(X86_XCPT_TS)
4739	\| RT_BIT(X86_XCPT_UD)
4740	\| RT_BIT(X86_XCPT_NP)
4741	\| RT_BIT(X86_XCPT_SS)
4742	\| RT_BIT(X86_XCPT_GP)
4743	\| RT_BIT(X86_XCPT_PF)
4744	\| RT_BIT(X86_XCPT_MF)
4745	;
4746	#elif defined(HMVMX_ALWAYS_TRAP_PF)
4747	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
4748	#endif
4749	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
4750	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
4751	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
4752
4753	/* Apply the fixed CR0 bits and enable caching. */
4754	u64GuestCr0 \|= fSetCr0;
4755	u64GuestCr0 &= fZapCr0;
4756	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4757
4758	/* Commit the CR0 and related fields to the guest VMCS. */
4759	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCr0);
4760	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4761	if (uProcCtls != pVmcsInfo->u32ProcCtls)
4762	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4763	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
4764	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
4765	AssertRCReturn(rc, rc);
4766
4767	/* Update our caches. */
4768	pVmcsInfo->u32ProcCtls = uProcCtls;
4769	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
4770
4771	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4772	}
4773	else
4774	{
4775	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
4776	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4777	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4778	uint64_t const u64ShadowCr0 = pVmcsNstGst->u64Cr0ReadShadow.u;
4779	Assert(!RT_HI_U32(u64GuestCr0));
4780	Assert(u64GuestCr0 & X86_CR0_NE);
4781
4782	/* Apply the fixed CR0 bits and enable caching. */
4783	u64GuestCr0 \|= fSetCr0;
4784	u64GuestCr0 &= fZapCr0;
4785	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4786
4787	/* Commit the CR0 and CR0 read shadow to the nested-guest VMCS. */
4788	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCr0);
4789	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4790	AssertRCReturn(rc, rc);
4791
4792	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4793	}
4794
4795	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
4796	}
4797
4798	return VINF_SUCCESS;
4799	}
4800
4801
4802	/**
4803	* Exports the guest control registers (CR3, CR4) into the guest-state area
4804	* in the VMCS.
4805	*
4806	* @returns VBox strict status code.
4807	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
4808	* without unrestricted guest access and the VMMDev is not presently
4809	* mapped (e.g. EFI32).
4810	*
4811	* @param pVCpu The cross context virtual CPU structure.
4812	* @param pVmxTransient The VMX-transient structure.
4813	*
4814	* @remarks No-long-jump zone!!!
4815	*/
4816	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4817	{
4818	int rc = VINF_SUCCESS;
4819	PVM pVM = pVCpu->CTX_SUFF(pVM);
4820
4821	/*
4822	* Guest CR2.
4823	* It's always loaded in the assembler code. Nothing to do here.
4824	*/
4825
4826	/*
4827	* Guest CR3.
4828	*/
4829	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
4830	{
4831	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
4832
4833	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
4834	if (pVM->hm.s.fNestedPaging)
4835	{
4836	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4837	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
4838
4839	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
4840	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
4841	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
4842	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
4843
4844	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
4845	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
4846	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
4847
4848	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
4849	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
4850	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
4851	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
4852	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
4853	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
4854	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
4855
4856	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
4857	AssertRCReturn(rc, rc);
4858
4859	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4860	if ( pVM->hm.s.vmx.fUnrestrictedGuest
4861	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
4862	{
4863	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
4864	if (CPUMIsGuestInPAEModeEx(pCtx))
4865	{
4866	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
4867	AssertRCReturn(rc, rc);
4868	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
4869	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
4870	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
4871	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
4872	AssertRCReturn(rc, rc);
4873	}
4874
4875	/*
4876	* The guest's view of its CR3 is unblemished with nested paging when the
4877	* guest is using paging or we have unrestricted guest execution to handle
4878	* the guest when it's not using paging.
4879	*/
4880	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
4881	GCPhysGuestCR3 = pCtx->cr3;
4882	}
4883	else
4884	{
4885	/*
4886	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
4887	* thinks it accesses physical memory directly, we use our identity-mapped
4888	* page table to map guest-linear to guest-physical addresses. EPT takes care
4889	* of translating it to host-physical addresses.
4890	*/
4891	RTGCPHYS GCPhys;
4892	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4893
4894	/* We obtain it here every time as the guest could have relocated this PCI region. */
4895	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4896	if (RT_SUCCESS(rc))
4897	{ /* likely */ }
4898	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4899	{
4900	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
4901	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4902	}
4903	else
4904	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4905
4906	GCPhysGuestCR3 = GCPhys;
4907	}
4908
4909	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
4910	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4911	AssertRCReturn(rc, rc);
4912	}
4913	else
4914	{
4915	/* Non-nested paging case, just use the hypervisor's CR3. */
4916	RTHCPHYS const HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4917
4918	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
4919	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4920	AssertRCReturn(rc, rc);
4921	}
4922
4923	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
4924	}
4925
4926	/*
4927	* Guest CR4.
4928	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4929	*/
4930	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
4931	{
4932	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4933	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4934	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
4935
4936	/*
4937	* Figure out fixed CR4 bits in VMX operation.
4938	*/
4939	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
4940	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
4941
4942	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
4943	uint64_t u64GuestCr4 = pCtx->cr4;
4944	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest ? pCtx->cr4 : pVmcsNstGst->u64Cr4ReadShadow.u;
4945	Assert(!RT_HI_U32(u64GuestCr4));
4946
4947	/*
4948	* Setup VT-x's view of the guest CR4.
4949	*
4950	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
4951	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
4952	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4953	*
4954	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4955	*/
4956	if (pVmcsInfo->RealMode.fRealOnV86Active)
4957	{
4958	Assert(pVM->hm.s.vmx.pRealModeTSS);
4959	Assert(PDMVmmDevHeapIsEnabled(pVM));
4960	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
4961	}
4962
4963	if (pVM->hm.s.fNestedPaging)
4964	{
4965	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
4966	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
4967	{
4968	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4969	u64GuestCr4 \|= X86_CR4_PSE;
4970	/* Our identity mapping is a 32-bit page directory. */
4971	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
4972	}
4973	/* else use guest CR4.*/
4974	}
4975	else
4976	{
4977	Assert(!pVmxTransient->fIsNestedGuest);
4978
4979	/*
4980	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4981	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4982	*/
4983	switch (pVCpu->hm.s.enmShadowMode)
4984	{
4985	case PGMMODE_REAL: /* Real-mode. */
4986	case PGMMODE_PROTECTED: /* Protected mode without paging. */
4987	case PGMMODE_32_BIT: /* 32-bit paging. */
4988	{
4989	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
4990	break;
4991	}
4992
4993	case PGMMODE_PAE: /* PAE paging. */
4994	case PGMMODE_PAE_NX: /* PAE paging with NX. */
4995	{
4996	u64GuestCr4 \|= X86_CR4_PAE;
4997	break;
4998	}
4999
5000	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5001	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5002	#ifdef VBOX_ENABLE_64_BITS_GUESTS
5003	break;
5004	#endif
5005	default:
5006	AssertFailed();
5007	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5008	}
5009	}
5010
5011	/* Apply the fixed CR4 bits (mainly CR4.VMXE). */
5012	u64GuestCr4 \|= fSetCr4;
5013	u64GuestCr4 &= fZapCr4;
5014
5015	/* Commit the CR4 and CR4 read shadow to the guest VMCS. */
5016	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR4, u64GuestCr4);
5017	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
5018	AssertRCReturn(rc, rc);
5019
5020	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5021	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5022
5023	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5024
5025	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5026	}
5027	return rc;
5028	}
5029
5030
5031	/**
5032	* Exports the guest debug registers into the guest-state area in the VMCS.
5033	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5034	*
5035	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5036	*
5037	* @returns VBox status code.
5038	* @param pVCpu The cross context virtual CPU structure.
5039	* @param pVmxTransient The VMX-transient structure.
5040	*
5041	* @remarks No-long-jump zone!!!
5042	*/
5043	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5044	{
5045	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5046
5047	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5048	* stepping. */
5049	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5050	if (pVmxTransient->fIsNestedGuest)
5051	{
5052	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5053	AssertRCReturn(rc, rc);
5054	return VINF_SUCCESS;
5055	}
5056
5057	#ifdef VBOX_STRICT
5058	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5059	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5060	{
5061	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5062	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5063	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5064	}
5065	#endif
5066
5067	bool fSteppingDB = false;
5068	bool fInterceptMovDRx = false;
5069	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5070	if (pVCpu->hm.s.fSingleInstruction)
5071	{
5072	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5073	PVM pVM = pVCpu->CTX_SUFF(pVM);
5074	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5075	{
5076	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5077	Assert(fSteppingDB == false);
5078	}
5079	else
5080	{
5081	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5082	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5083	pVCpu->hm.s.fClearTrapFlag = true;
5084	fSteppingDB = true;
5085	}
5086	}
5087
5088	uint32_t u32GuestDr7;
5089	if ( fSteppingDB
5090	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5091	{
5092	/*
5093	* Use the combined guest and host DRx values found in the hypervisor register set
5094	* because the hypervisor debugger has breakpoints active or someone is single stepping
5095	* on the host side without a monitor trap flag.
5096	*
5097	* Note! DBGF expects a clean DR6 state before executing guest code.
5098	*/
5099	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5100	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5101	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
5102	{
5103	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5104	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
5105	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
5106	}
5107	else
5108	#endif
5109	if (!CPUMIsHyperDebugStateActive(pVCpu))
5110	{
5111	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5112	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5113	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5114	}
5115
5116	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5117	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
5118	pVCpu->hm.s.fUsingHyperDR7 = true;
5119	fInterceptMovDRx = true;
5120	}
5121	else
5122	{
5123	/*
5124	* If the guest has enabled debug registers, we need to load them prior to
5125	* executing guest code so they'll trigger at the right time.
5126	*/
5127	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5128	{
5129	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5130	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5131	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
5132	{
5133	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5134	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
5135	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
5136	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5137	}
5138	else
5139	#endif
5140	if (!CPUMIsGuestDebugStateActive(pVCpu))
5141	{
5142	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5143	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5144	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5145	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5146	}
5147	Assert(!fInterceptMovDRx);
5148	}
5149	/*
5150	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5151	* must intercept #DB in order to maintain a correct DR6 guest value, and
5152	* because we need to intercept it to prevent nested #DBs from hanging the
5153	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5154	*/
5155	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5156	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
5157	&& !CPUMIsGuestDebugStateActive(pVCpu))
5158	#else
5159	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5160	#endif
5161	{
5162	fInterceptMovDRx = true;
5163	}
5164
5165	/* Update DR7 with the actual guest value. */
5166	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5167	pVCpu->hm.s.fUsingHyperDR7 = false;
5168	}
5169
5170	if (fInterceptMovDRx)
5171	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5172	else
5173	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5174
5175	/*
5176	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5177	* monitor-trap flag and update our cache.
5178	*/
5179	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5180	{
5181	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5182	AssertRCReturn(rc2, rc2);
5183	pVmcsInfo->u32ProcCtls = uProcCtls;
5184	}
5185
5186	/*
5187	* Update guest DR7.
5188	*/
5189	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
5190	AssertRCReturn(rc, rc);
5191
5192	/*
5193	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
5194	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
5195	*
5196	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5197	*/
5198	if (fSteppingDB)
5199	{
5200	Assert(pVCpu->hm.s.fSingleInstruction);
5201	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
5202
5203	uint32_t fIntrState = 0;
5204	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
5205	AssertRCReturn(rc, rc);
5206
5207	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5208	{
5209	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5210	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5211	AssertRCReturn(rc, rc);
5212	}
5213	}
5214
5215	return VINF_SUCCESS;
5216	}
5217
5218
5219	#ifdef VBOX_STRICT
5220	/**
5221	* Strict function to validate segment registers.
5222	*
5223	* @param pVCpu The cross context virtual CPU structure.
5224	* @param pVmcsInfo The VMCS info. object.
5225	*
5226	* @remarks Will import guest CR0 on strict builds during validation of
5227	* segments.
5228	*/
5229	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
5230	{
5231	/*
5232	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5233	*
5234	* The reason we check for attribute value 0 in this function and not just the unusable bit is
5235	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
5236	* unusable bit and doesn't change the guest-context value.
5237	*/
5238	PVM pVM = pVCpu->CTX_SUFF(pVM);
5239	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5240	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
5241	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
5242	&& ( !CPUMIsGuestInRealModeEx(pCtx)
5243	&& !CPUMIsGuestInV86ModeEx(pCtx)))
5244	{
5245	/* Protected mode checks */
5246	/* CS */
5247	Assert(pCtx->cs.Attr.n.u1Present);
5248	Assert(!(pCtx->cs.Attr.u & 0xf00));
5249	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
5250	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5251	\|\| !(pCtx->cs.Attr.n.u1Granularity));
5252	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
5253	\|\| (pCtx->cs.Attr.n.u1Granularity));
5254	/* CS cannot be loaded with NULL in protected mode. */
5255	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
5256	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5257	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
5258	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5259	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
5260	else
5261	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
5262	/* SS */
5263	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5264	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
5265	if ( !(pCtx->cr0 & X86_CR0_PE)
5266	\|\| pCtx->cs.Attr.n.u4Type == 3)
5267	{
5268	Assert(!pCtx->ss.Attr.n.u2Dpl);
5269	}
5270	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
5271	{
5272	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5273	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
5274	Assert(pCtx->ss.Attr.n.u1Present);
5275	Assert(!(pCtx->ss.Attr.u & 0xf00));
5276	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
5277	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
5278	\|\| !(pCtx->ss.Attr.n.u1Granularity));
5279	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
5280	\|\| (pCtx->ss.Attr.n.u1Granularity));
5281	}
5282	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
5283	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
5284	{
5285	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5286	Assert(pCtx->ds.Attr.n.u1Present);
5287	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
5288	Assert(!(pCtx->ds.Attr.u & 0xf00));
5289	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
5290	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
5291	\|\| !(pCtx->ds.Attr.n.u1Granularity));
5292	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
5293	\|\| (pCtx->ds.Attr.n.u1Granularity));
5294	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5295	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
5296	}
5297	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
5298	{
5299	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5300	Assert(pCtx->es.Attr.n.u1Present);
5301	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
5302	Assert(!(pCtx->es.Attr.u & 0xf00));
5303	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
5304	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
5305	\|\| !(pCtx->es.Attr.n.u1Granularity));
5306	Assert( !(pCtx->es.u32Limit & 0xfff00000)
5307	\|\| (pCtx->es.Attr.n.u1Granularity));
5308	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5309	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
5310	}
5311	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
5312	{
5313	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5314	Assert(pCtx->fs.Attr.n.u1Present);
5315	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
5316	Assert(!(pCtx->fs.Attr.u & 0xf00));
5317	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
5318	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
5319	\|\| !(pCtx->fs.Attr.n.u1Granularity));
5320	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
5321	\|\| (pCtx->fs.Attr.n.u1Granularity));
5322	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5323	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5324	}
5325	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
5326	{
5327	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5328	Assert(pCtx->gs.Attr.n.u1Present);
5329	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
5330	Assert(!(pCtx->gs.Attr.u & 0xf00));
5331	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
5332	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
5333	\|\| !(pCtx->gs.Attr.n.u1Granularity));
5334	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
5335	\|\| (pCtx->gs.Attr.n.u1Granularity));
5336	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5337	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5338	}
5339	/* 64-bit capable CPUs. */
5340	# if HC_ARCH_BITS == 64
5341	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5342	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
5343	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
5344	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
5345	# endif
5346	}
5347	else if ( CPUMIsGuestInV86ModeEx(pCtx)
5348	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
5349	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
5350	{
5351	/* Real and v86 mode checks. */
5352	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
5353	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
5354	if (pVmcsInfo->RealMode.fRealOnV86Active)
5355	{
5356	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
5357	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
5358	}
5359	else
5360	{
5361	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
5362	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
5363	}
5364
5365	/* CS */
5366	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
5367	Assert(pCtx->cs.u32Limit == 0xffff);
5368	Assert(u32CSAttr == 0xf3);
5369	/* SS */
5370	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
5371	Assert(pCtx->ss.u32Limit == 0xffff);
5372	Assert(u32SSAttr == 0xf3);
5373	/* DS */
5374	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
5375	Assert(pCtx->ds.u32Limit == 0xffff);
5376	Assert(u32DSAttr == 0xf3);
5377	/* ES */
5378	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
5379	Assert(pCtx->es.u32Limit == 0xffff);
5380	Assert(u32ESAttr == 0xf3);
5381	/* FS */
5382	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
5383	Assert(pCtx->fs.u32Limit == 0xffff);
5384	Assert(u32FSAttr == 0xf3);
5385	/* GS */
5386	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
5387	Assert(pCtx->gs.u32Limit == 0xffff);
5388	Assert(u32GSAttr == 0xf3);
5389	/* 64-bit capable CPUs. */
5390	# if HC_ARCH_BITS == 64
5391	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5392	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
5393	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
5394	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
5395	# endif
5396	}
5397	}
5398	#endif /* VBOX_STRICT */
5399
5400
5401	/**
5402	* Exports a guest segment register into the guest-state area in the VMCS.
5403	*
5404	* @returns VBox status code.
5405	* @param pVCpu The cross context virtual CPU structure.
5406	* @param pVmcsInfo The VMCS info. object.
5407	* @param iSegReg The segment register number (X86_SREG_XXX).
5408	* @param pSelReg Pointer to the segment selector.
5409	*
5410	* @remarks No-long-jump zone!!!
5411	*/
5412	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
5413	{
5414	Assert(iSegReg < X86_SREG_COUNT);
5415	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
5416	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
5417	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
5418	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
5419
5420	uint32_t u32Access = pSelReg->Attr.u;
5421	if (pVmcsInfo->RealMode.fRealOnV86Active)
5422	{
5423	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
5424	u32Access = 0xf3;
5425	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5426	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5427	RT_NOREF_PV(pVCpu);
5428	}
5429	else
5430	{
5431	/*
5432	* The way to differentiate between whether this is really a null selector or was just
5433	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
5434	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
5435	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
5436	* NULL selectors loaded in protected-mode have their attribute as 0.
5437	*/
5438	if (!u32Access)
5439	u32Access = X86DESCATTR_UNUSABLE;
5440	}
5441
5442	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
5443	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
5444	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
5445
5446	/*
5447	* Commit it to the VMCS.
5448	*/
5449	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
5450	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
5451	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
5452	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
5453	AssertRCReturn(rc, rc);
5454	return rc;
5455	}
5456
5457
5458	/**
5459	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
5460	* area in the VMCS.
5461	*
5462	* @returns VBox status code.
5463	* @param pVCpu The cross context virtual CPU structure.
5464	* @param pVmxTransient The VMX-transient structure.
5465	*
5466	* @remarks Will import guest CR0 on strict builds during validation of
5467	* segments.
5468	* @remarks No-long-jump zone!!!
5469	*/
5470	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5471	{
5472	int rc = VERR_INTERNAL_ERROR_5;
5473	PVM pVM = pVCpu->CTX_SUFF(pVM);
5474	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5475	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5476
5477	/*
5478	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
5479	*/
5480	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
5481	{
5482	#ifdef VBOX_WITH_REM
5483	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
5484	{
5485	Assert(!pVmxTransient->fIsNestedGuest);
5486	Assert(pVM->hm.s.vmx.pRealModeTSS);
5487	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
5488	if ( pVmcsInfo->fWasInRealMode
5489	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
5490	{
5491	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
5492	in real-mode (e.g. OpenBSD 4.0) */
5493	REMFlushTBs(pVM);
5494	Log4Func(("Switch to protected mode detected!\n"));
5495	pVmcsInfo->fWasInRealMode = false;
5496	}
5497	}
5498	#endif
5499	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
5500	{
5501	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
5502	if (pVmcsInfo->RealMode.fRealOnV86Active)
5503	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
5504	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
5505	AssertRCReturn(rc, rc);
5506	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
5507	}
5508
5509	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
5510	{
5511	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
5512	if (pVmcsInfo->RealMode.fRealOnV86Active)
5513	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
5514	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
5515	AssertRCReturn(rc, rc);
5516	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
5517	}
5518
5519	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
5520	{
5521	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
5522	if (pVmcsInfo->RealMode.fRealOnV86Active)
5523	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
5524	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
5525	AssertRCReturn(rc, rc);
5526	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
5527	}
5528
5529	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
5530	{
5531	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
5532	if (pVmcsInfo->RealMode.fRealOnV86Active)
5533	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
5534	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
5535	AssertRCReturn(rc, rc);
5536	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
5537	}
5538
5539	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
5540	{
5541	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
5542	if (pVmcsInfo->RealMode.fRealOnV86Active)
5543	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
5544	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
5545	AssertRCReturn(rc, rc);
5546	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
5547	}
5548
5549	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
5550	{
5551	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
5552	if (pVmcsInfo->RealMode.fRealOnV86Active)
5553	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
5554	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
5555	AssertRCReturn(rc, rc);
5556	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
5557	}
5558
5559	#ifdef VBOX_STRICT
5560	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
5561	#endif
5562	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
5563	pCtx->cs.Attr.u));
5564	}
5565
5566	/*
5567	* Guest TR.
5568	*/
5569	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
5570	{
5571	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
5572
5573	/*
5574	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
5575	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
5576	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
5577	*/
5578	uint16_t u16Sel;
5579	uint32_t u32Limit;
5580	uint64_t u64Base;
5581	uint32_t u32AccessRights;
5582	if (!pVmcsInfo->RealMode.fRealOnV86Active)
5583	{
5584	u16Sel = pCtx->tr.Sel;
5585	u32Limit = pCtx->tr.u32Limit;
5586	u64Base = pCtx->tr.u64Base;
5587	u32AccessRights = pCtx->tr.Attr.u;
5588	}
5589	else
5590	{
5591	Assert(!pVmxTransient->fIsNestedGuest);
5592	Assert(pVM->hm.s.vmx.pRealModeTSS);
5593	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
5594
5595	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
5596	RTGCPHYS GCPhys;
5597	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
5598	AssertRCReturn(rc, rc);
5599
5600	X86DESCATTR DescAttr;
5601	DescAttr.u = 0;
5602	DescAttr.n.u1Present = 1;
5603	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
5604
5605	u16Sel = 0;
5606	u32Limit = HM_VTX_TSS_SIZE;
5607	u64Base = GCPhys;
5608	u32AccessRights = DescAttr.u;
5609	}
5610
5611	/* Validate. */
5612	Assert(!(u16Sel & RT_BIT(2)));
5613	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
5614	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
5615	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
5616	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
5617	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
5618	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
5619	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
5620	Assert( (u32Limit & 0xfff) == 0xfff
5621	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
5622	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
5623	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
5624
5625	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
5626	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
5627	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
5628	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
5629	AssertRCReturn(rc, rc);
5630
5631	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
5632	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
5633	}
5634
5635	/*
5636	* Guest GDTR.
5637	*/
5638	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
5639	{
5640	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
5641
5642	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
5643	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
5644	AssertRCReturn(rc, rc);
5645
5646	/* Validate. */
5647	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5648
5649	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
5650	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
5651	}
5652
5653	/*
5654	* Guest LDTR.
5655	*/
5656	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
5657	{
5658	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
5659
5660	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
5661	uint32_t u32Access;
5662	if ( !pVmxTransient->fIsNestedGuest
5663	&& !pCtx->ldtr.Attr.u)
5664	u32Access = X86DESCATTR_UNUSABLE;
5665	else
5666	u32Access = pCtx->ldtr.Attr.u;
5667
5668	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
5669	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
5670	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
5671	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
5672	AssertRCReturn(rc, rc);
5673
5674	/* Validate. */
5675	if (!(u32Access & X86DESCATTR_UNUSABLE))
5676	{
5677	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
5678	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
5679	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
5680	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
5681	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
5682	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
5683	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
5684	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
5685	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
5686	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
5687	}
5688
5689	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
5690	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
5691	}
5692
5693	/*
5694	* Guest IDTR.
5695	*/
5696	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
5697	{
5698	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
5699
5700	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
5701	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
5702	AssertRCReturn(rc, rc);
5703
5704	/* Validate. */
5705	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5706
5707	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
5708	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
5709	}
5710
5711	return VINF_SUCCESS;
5712	}
5713
5714
5715	/**
5716	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
5717	* areas.
5718	*
5719	* These MSRs will automatically be loaded to the host CPU on every successful
5720	* VM-entry and stored from the host CPU on every successful VM-exit.
5721	*
5722	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
5723	* actual host MSR values are not- updated here for performance reasons. See
5724	* hmR0VmxExportHostMsrs().
5725	*
5726	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
5727	*
5728	* @returns VBox status code.
5729	* @param pVCpu The cross context virtual CPU structure.
5730	* @param pVmxTransient The VMX-transient structure.
5731	*
5732	* @remarks No-long-jump zone!!!
5733	*/
5734	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5735	{
5736	AssertPtr(pVCpu);
5737	AssertPtr(pVmxTransient);
5738
5739	PVM pVM = pVCpu->CTX_SUFF(pVM);
5740	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5741
5742	/*
5743	* MSRs that we use the auto-load/store MSR area in the VMCS.
5744	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
5745	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
5746	*
5747	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
5748	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
5749	* emulation, nothing to do here.
5750	*/
5751	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
5752	{
5753	if ( !pVmxTransient->fIsNestedGuest
5754	&& pVM->hm.s.fAllow64BitGuests)
5755	{
5756	#if HC_ARCH_BITS == 32
5757	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
5758	Assert(!pVmxTransient->fIsNestedGuest);
5759
5760	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
5761	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
5762	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
5763	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
5764	AssertRCReturn(rc, rc);
5765	#endif
5766	}
5767	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
5768	}
5769
5770	/*
5771	* Guest Sysenter MSRs.
5772	*/
5773	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
5774	{
5775	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
5776
5777	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
5778	{
5779	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
5780	AssertRCReturn(rc, rc);
5781	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
5782	}
5783
5784	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
5785	{
5786	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
5787	AssertRCReturn(rc, rc);
5788	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
5789	}
5790
5791	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
5792	{
5793	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
5794	AssertRCReturn(rc, rc);
5795	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
5796	}
5797	}
5798
5799	/*
5800	* Guest/host EFER MSR.
5801	*/
5802	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
5803	{
5804	/* Whether we are using the VMCS to swap the EFER MSR must have been
5805	determined earlier while exporting VM-entry/VM-exit controls. */
5806	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
5807	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
5808
5809	if (hmR0VmxShouldSwapEferMsr(pVCpu))
5810	{
5811	/*
5812	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
5813	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
5814	*/
5815	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
5816	{
5817	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
5818	AssertRCReturn(rc, rc);
5819	}
5820	else
5821	{
5822	/*
5823	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
5824	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
5825	*/
5826	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
5827	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5828	AssertRCReturn(rc, rc);
5829	}
5830	}
5831	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
5832	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
5833
5834	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
5835	}
5836
5837	/*
5838	* Other MSRs.
5839	* Speculation Control (R/W).
5840	*/
5841	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
5842	{
5843	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
5844	if (pVM->cpum.ro.GuestFeatures.fIbrs)
5845	{
5846	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
5847	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5848	AssertRCReturn(rc, rc);
5849	}
5850	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
5851	}
5852
5853	return VINF_SUCCESS;
5854	}
5855
5856
5857	/**
5858	* Selects up the appropriate function to run guest code.
5859	*
5860	* @returns VBox status code.
5861	* @param pVCpu The cross context virtual CPU structure.
5862	* @param pVmxTransient The VMX-transient structure.
5863	*
5864	* @remarks No-long-jump zone!!!
5865	*/
5866	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5867	{
5868	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5869	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5870
5871	if (CPUMIsGuestInLongModeEx(pCtx))
5872	{
5873	#ifndef VBOX_ENABLE_64_BITS_GUESTS
5874	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5875	#endif
5876	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
5877	#if HC_ARCH_BITS == 32
5878	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
5879	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
5880	{
5881	#ifdef VBOX_STRICT
5882	if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5883	{
5884	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5885	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5886	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5887	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5888	("fCtxChanged=%#RX64\n", fCtxChanged));
5889	}
5890	#endif
5891	pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
5892
5893	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
5894	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
5895	pVmcsInfo->fSwitchedTo64on32 = true;
5896	Log4Func(("Selected 64-bit switcher\n"));
5897	}
5898	#else
5899	/* 64-bit host. */
5900	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
5901	#endif
5902	}
5903	else
5904	{
5905	/* Guest is not in long mode, use the 32-bit handler. */
5906	#if HC_ARCH_BITS == 32
5907	if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
5908	&& !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
5909	&& pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5910	{
5911	# ifdef VBOX_STRICT
5912	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5913	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5914	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5915	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5916	("fCtxChanged=%#RX64\n", fCtxChanged));
5917	# endif
5918	}
5919	# ifdef VBOX_ENABLE_64_BITS_GUESTS
5920	/*
5921	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
5922	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
5923	* switcher flag now because we know the guest is in a sane state where it's safe
5924	* to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
5925	* the much faster 32-bit switcher again.
5926	*/
5927	if (!pVmcsInfo->fSwitchedTo64on32)
5928	{
5929	if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
5930	Log4Func(("Selected 32-bit switcher\n"));
5931	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5932	}
5933	else
5934	{
5935	Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
5936	if ( pVmcsInfo->RealMode.fRealOnV86Active
5937	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
5938	{
5939	pVmcsInfo->fSwitchedTo64on32 = false;
5940	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5941	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
5942	\| HM_CHANGED_VMX_ENTRY_EXIT_CTLS
5943	\| HM_CHANGED_HOST_CONTEXT);
5944	Log4Func(("Selected 32-bit switcher (safe)\n"));
5945	}
5946	}
5947	# else
5948	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5949	# endif
5950	#else
5951	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5952	#endif
5953	}
5954	Assert(pVmcsInfo->pfnStartVM);
5955	return VINF_SUCCESS;
5956	}
5957
5958
5959	/**
5960	* Wrapper for running the guest code in VT-x.
5961	*
5962	* @returns VBox status code, no informational status codes.
5963	* @param pVCpu The cross context virtual CPU structure.
5964	* @param pVmxTransient The VMX-transient structure.
5965	*
5966	* @remarks No-long-jump zone!!!
5967	*/
5968	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5969	{
5970	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
5971	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5972	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
5973
5974	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
5975
5976	/*
5977	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
5978	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
5979	* callee-saved and thus the need for this XMM wrapper.
5980	*
5981	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
5982	*/
5983	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5984	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
5985	PVM pVM = pVCpu->CTX_SUFF(pVM);
5986	#ifdef VBOX_WITH_KERNEL_USING_XMM
5987	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
5988	#else
5989	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
5990	#endif
5991	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5992	return rc;
5993	}
5994
5995
5996	/**
5997	* Reports world-switch error and dumps some useful debug info.
5998	*
5999	* @param pVCpu The cross context virtual CPU structure.
6000	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6001	* @param pVmxTransient The VMX-transient structure (only
6002	* exitReason updated).
6003	*/
6004	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6005	{
6006	Assert(pVCpu);
6007	Assert(pVmxTransient);
6008	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6009
6010	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6011	switch (rcVMRun)
6012	{
6013	case VERR_VMX_INVALID_VMXON_PTR:
6014	AssertFailed();
6015	break;
6016	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6017	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6018	{
6019	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6020	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6021	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
6022	AssertRC(rc);
6023
6024	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6025	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6026	Cannot do it here as we may have been long preempted. */
6027
6028	#ifdef VBOX_STRICT
6029	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6030	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6031	pVmxTransient->uExitReason));
6032	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6033	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6034	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6035	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6036	else
6037	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6038	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6039	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6040
6041	/* VMX control bits. */
6042	uint32_t u32Val;
6043	uint64_t u64Val;
6044	RTHCUINTREG uHCReg;
6045	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
6046	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
6047	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
6048	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
6049	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
6050	{
6051	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
6052	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
6053	}
6054	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
6055	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
6056	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
6057	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
6058	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
6059	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
6060	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
6061	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
6062	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
6063	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
6064	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
6065	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
6066	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
6067	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
6068	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
6069	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
6070	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6071	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
6072	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6073	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
6074	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
6075	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
6076	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
6077	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
6078	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
6079	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
6080	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
6081	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
6082	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
6083	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6084	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
6085	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
6086	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
6087	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6088	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6089	{
6090	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
6091	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
6092	}
6093
6094	/* Guest bits. */
6095	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
6096	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
6097	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
6098	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
6099	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
6100	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
6101	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
6102	{
6103	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
6104	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
6105	}
6106
6107	/* Host bits. */
6108	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
6109	Log4(("Host CR0 %#RHr\n", uHCReg));
6110	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
6111	Log4(("Host CR3 %#RHr\n", uHCReg));
6112	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
6113	Log4(("Host CR4 %#RHr\n", uHCReg));
6114
6115	RTGDTR HostGdtr;
6116	PCX86DESCHC pDesc;
6117	ASMGetGDTR(&HostGdtr);
6118	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
6119	Log4(("Host CS %#08x\n", u32Val));
6120	if (u32Val < HostGdtr.cbGdt)
6121	{
6122	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6123	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
6124	}
6125
6126	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
6127	Log4(("Host DS %#08x\n", u32Val));
6128	if (u32Val < HostGdtr.cbGdt)
6129	{
6130	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6131	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
6132	}
6133
6134	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
6135	Log4(("Host ES %#08x\n", u32Val));
6136	if (u32Val < HostGdtr.cbGdt)
6137	{
6138	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6139	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
6140	}
6141
6142	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
6143	Log4(("Host FS %#08x\n", u32Val));
6144	if (u32Val < HostGdtr.cbGdt)
6145	{
6146	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6147	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
6148	}
6149
6150	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
6151	Log4(("Host GS %#08x\n", u32Val));
6152	if (u32Val < HostGdtr.cbGdt)
6153	{
6154	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6155	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
6156	}
6157
6158	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
6159	Log4(("Host SS %#08x\n", u32Val));
6160	if (u32Val < HostGdtr.cbGdt)
6161	{
6162	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6163	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
6164	}
6165
6166	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
6167	Log4(("Host TR %#08x\n", u32Val));
6168	if (u32Val < HostGdtr.cbGdt)
6169	{
6170	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6171	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
6172	}
6173
6174	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
6175	Log4(("Host TR Base %#RHv\n", uHCReg));
6176	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
6177	Log4(("Host GDTR Base %#RHv\n", uHCReg));
6178	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
6179	Log4(("Host IDTR Base %#RHv\n", uHCReg));
6180	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
6181	Log4(("Host SYSENTER CS %#08x\n", u32Val));
6182	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
6183	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
6184	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
6185	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
6186	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
6187	Log4(("Host RSP %#RHv\n", uHCReg));
6188	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
6189	Log4(("Host RIP %#RHv\n", uHCReg));
6190	# if HC_ARCH_BITS == 64
6191	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6192	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6193	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6194	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6195	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6196	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6197	# endif
6198	#endif /* VBOX_STRICT */
6199	break;
6200	}
6201
6202	default:
6203	/* Impossible */
6204	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6205	break;
6206	}
6207	}
6208
6209
6210	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
6211	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
6212	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
6213	#endif
6214	#ifdef VBOX_STRICT
6215	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
6216	{
6217	switch (idxField)
6218	{
6219	case VMX_VMCS_GUEST_RIP:
6220	case VMX_VMCS_GUEST_RSP:
6221	case VMX_VMCS_GUEST_SYSENTER_EIP:
6222	case VMX_VMCS_GUEST_SYSENTER_ESP:
6223	case VMX_VMCS_GUEST_GDTR_BASE:
6224	case VMX_VMCS_GUEST_IDTR_BASE:
6225	case VMX_VMCS_GUEST_CS_BASE:
6226	case VMX_VMCS_GUEST_DS_BASE:
6227	case VMX_VMCS_GUEST_ES_BASE:
6228	case VMX_VMCS_GUEST_FS_BASE:
6229	case VMX_VMCS_GUEST_GS_BASE:
6230	case VMX_VMCS_GUEST_SS_BASE:
6231	case VMX_VMCS_GUEST_LDTR_BASE:
6232	case VMX_VMCS_GUEST_TR_BASE:
6233	case VMX_VMCS_GUEST_CR3:
6234	return true;
6235	}
6236	return false;
6237	}
6238
6239	static bool hmR0VmxIsValidReadField(uint32_t idxField)
6240	{
6241	switch (idxField)
6242	{
6243	/* Read-only fields. */
6244	case VMX_VMCS_RO_EXIT_QUALIFICATION:
6245	return true;
6246	}
6247	/* Remaining readable fields should also be writable. */
6248	return hmR0VmxIsValidWriteField(idxField);
6249	}
6250	#endif /* VBOX_STRICT */
6251
6252
6253	/**
6254	* Initialize the VMCS-Read cache.
6255	*
6256	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
6257	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
6258	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
6259	* (those that have a 32-bit FULL & HIGH part).
6260	*
6261	* @param pVCpu The cross context virtual CPU structure.
6262	*/
6263	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
6264	{
6265	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
6266	do { \
6267	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
6268	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
6269	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
6270	++cReadFields; \
6271	} while (0)
6272
6273	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6274	uint32_t cReadFields = 0;
6275
6276	/*
6277	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
6278	* and serve to indicate exceptions to the rules.
6279	*/
6280
6281	/* Guest-natural selector base fields. */
6282	#if 0
6283	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
6284	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
6285	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
6286	#endif
6287	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
6288	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
6289	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
6290	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
6291	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
6292	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
6293	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
6294	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
6295	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
6296	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
6297	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
6298	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
6299	#if 0
6300	/* Unused natural width guest-state fields. */
6301	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
6302	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
6303	#endif
6304	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
6305	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
6306
6307	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
6308	these 64-bit fields (using "FULL" and "HIGH" fields). */
6309	#if 0
6310	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
6311	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
6312	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
6313	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
6314	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
6315	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
6316	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
6317	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
6318	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
6319	#endif
6320
6321	/* Natural width guest-state fields. */
6322	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
6323	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
6324
6325	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6326	{
6327	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
6328	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
6329	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
6330	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
6331	}
6332	else
6333	{
6334	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
6335	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
6336	}
6337
6338	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
6339	}
6340
6341
6342	/**
6343	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
6344	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
6345	* darwin, running 64-bit guests).
6346	*
6347	* @returns VBox status code.
6348	* @param pVCpu The cross context virtual CPU structure.
6349	* @param idxField The VMCS field encoding.
6350	* @param u64Val 16, 32 or 64-bit value.
6351	*/
6352	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6353	{
6354	int rc;
6355	switch (idxField)
6356	{
6357	/*
6358	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
6359	*/
6360	/* 64-bit Control fields. */
6361	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
6362	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
6363	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
6364	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
6365	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
6366	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
6367	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
6368	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
6369	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
6370	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
6371	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
6372	case VMX_VMCS64_CTRL_EPTP_FULL:
6373	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
6374	/* 64-bit Guest-state fields. */
6375	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
6376	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
6377	case VMX_VMCS64_GUEST_PAT_FULL:
6378	case VMX_VMCS64_GUEST_EFER_FULL:
6379	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
6380	case VMX_VMCS64_GUEST_PDPTE0_FULL:
6381	case VMX_VMCS64_GUEST_PDPTE1_FULL:
6382	case VMX_VMCS64_GUEST_PDPTE2_FULL:
6383	case VMX_VMCS64_GUEST_PDPTE3_FULL:
6384	/* 64-bit Host-state fields. */
6385	case VMX_VMCS64_HOST_PAT_FULL:
6386	case VMX_VMCS64_HOST_EFER_FULL:
6387	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
6388	{
6389	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6390	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
6391	break;
6392	}
6393
6394	/*
6395	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
6396	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
6397	*/
6398	/* Natural-width Guest-state fields. */
6399	case VMX_VMCS_GUEST_CR3:
6400	case VMX_VMCS_GUEST_ES_BASE:
6401	case VMX_VMCS_GUEST_CS_BASE:
6402	case VMX_VMCS_GUEST_SS_BASE:
6403	case VMX_VMCS_GUEST_DS_BASE:
6404	case VMX_VMCS_GUEST_FS_BASE:
6405	case VMX_VMCS_GUEST_GS_BASE:
6406	case VMX_VMCS_GUEST_LDTR_BASE:
6407	case VMX_VMCS_GUEST_TR_BASE:
6408	case VMX_VMCS_GUEST_GDTR_BASE:
6409	case VMX_VMCS_GUEST_IDTR_BASE:
6410	case VMX_VMCS_GUEST_RSP:
6411	case VMX_VMCS_GUEST_RIP:
6412	case VMX_VMCS_GUEST_SYSENTER_ESP:
6413	case VMX_VMCS_GUEST_SYSENTER_EIP:
6414	{
6415	if (!(RT_HI_U32(u64Val)))
6416	{
6417	/* If this field is 64-bit, VT-x will zero out the top bits. */
6418	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6419	}
6420	else
6421	{
6422	/* Assert that only the 32->64 switcher case should ever come here. */
6423	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
6424	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
6425	}
6426	break;
6427	}
6428
6429	default:
6430	{
6431	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
6432	rc = VERR_INVALID_PARAMETER;
6433	break;
6434	}
6435	}
6436	AssertRCReturn(rc, rc);
6437	return rc;
6438	}
6439
6440
6441	/**
6442	* Queue up a VMWRITE by using the VMCS write cache.
6443	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
6444	*
6445	* @param pVCpu The cross context virtual CPU structure.
6446	* @param idxField The VMCS field encoding.
6447	* @param u64Val 16, 32 or 64-bit value.
6448	*/
6449	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6450	{
6451	AssertPtr(pVCpu);
6452	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6453
6454	AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
6455	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
6456
6457	/* Make sure there are no duplicates. */
6458	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
6459	{
6460	if (pCache->Write.aField[i] == idxField)
6461	{
6462	pCache->Write.aFieldVal[i] = u64Val;
6463	return VINF_SUCCESS;
6464	}
6465	}
6466
6467	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
6468	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
6469	pCache->Write.cValidEntries++;
6470	return VINF_SUCCESS;
6471	}
6472	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
6473
6474
6475	/**
6476	* Sets up the usage of TSC-offsetting and updates the VMCS.
6477	*
6478	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6479	* VMX-preemption timer.
6480	*
6481	* @returns VBox status code.
6482	* @param pVCpu The cross context virtual CPU structure.
6483	* @param pVmxTransient The VMX-transient structure.
6484	*
6485	* @remarks No-long-jump zone!!!
6486	*/
6487	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6488	{
6489	bool fOffsettedTsc;
6490	bool fParavirtTsc;
6491	uint64_t uTscOffset;
6492	PVM pVM = pVCpu->CTX_SUFF(pVM);
6493	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);;
6494
6495	if (pVM->hm.s.vmx.fUsePreemptTimer)
6496	{
6497	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6498
6499	/* Make sure the returned values have sane upper and lower boundaries. */
6500	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6501	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6502	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6503	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6504
6505	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6506	* preemption timers here. We probably need to clamp the preemption timer,
6507	* after converting the timer value to the host. */
6508	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6509	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6510	AssertRC(rc);
6511	}
6512	else
6513	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6514
6515	if (fParavirtTsc)
6516	{
6517	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6518	information before every VM-entry, hence disable it for performance sake. */
6519	#if 0
6520	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
6521	AssertRC(rc);
6522	#endif
6523	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
6524	}
6525
6526	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
6527	if ( fOffsettedTsc
6528	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
6529	{
6530	if (pVmxTransient->fIsNestedGuest)
6531	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
6532	if (pVmcsInfo->u64TscOffset != uTscOffset)
6533	{
6534	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
6535	AssertRC(rc);
6536	pVmcsInfo->u64TscOffset = uTscOffset;
6537	}
6538
6539	if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
6540	{
6541	uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
6542	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6543	AssertRC(rc);
6544	pVmcsInfo->u32ProcCtls = uProcCtls;
6545	}
6546	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
6547	}
6548	else
6549	{
6550	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
6551	if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
6552	{
6553	uProcCtls \|= VMX_PROC_CTLS_RDTSC_EXIT;
6554	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6555	AssertRC(rc);
6556	pVmcsInfo->u32ProcCtls = uProcCtls;
6557	}
6558	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
6559	}
6560	}
6561
6562
6563	/**
6564	* Gets the IEM exception flags for the specified vector and IDT vectoring /
6565	* VM-exit interruption info type.
6566	*
6567	* @returns The IEM exception flags.
6568	* @param uVector The event vector.
6569	* @param uVmxEventType The VMX event type.
6570	*
6571	* @remarks This function currently only constructs flags required for
6572	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
6573	* and CR2 aspects of an exception are not included).
6574	*/
6575	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
6576	{
6577	uint32_t fIemXcptFlags;
6578	switch (uVmxEventType)
6579	{
6580	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6581	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6582	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6583	break;
6584
6585	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6586	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6587	break;
6588
6589	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6590	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
6591	break;
6592
6593	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
6594	{
6595	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6596	if (uVector == X86_XCPT_BP)
6597	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
6598	else if (uVector == X86_XCPT_OF)
6599	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
6600	else
6601	{
6602	fIemXcptFlags = 0;
6603	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
6604	}
6605	break;
6606	}
6607
6608	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6609	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6610	break;
6611
6612	default:
6613	fIemXcptFlags = 0;
6614	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
6615	break;
6616	}
6617	return fIemXcptFlags;
6618	}
6619
6620
6621	/**
6622	* Sets an event as a pending event to be injected into the guest.
6623	*
6624	* @param pVCpu The cross context virtual CPU structure.
6625	* @param u32IntInfo The VM-entry interruption-information field.
6626	* @param cbInstr The VM-entry instruction length in bytes (for software
6627	* interrupts, exceptions and privileged software
6628	* exceptions).
6629	* @param u32ErrCode The VM-entry exception error code.
6630	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
6631	* page-fault.
6632	*/
6633	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
6634	RTGCUINTPTR GCPtrFaultAddress)
6635	{
6636	Assert(!pVCpu->hm.s.Event.fPending);
6637	pVCpu->hm.s.Event.fPending = true;
6638	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
6639	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
6640	pVCpu->hm.s.Event.cbInstr = cbInstr;
6641	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
6642	}
6643
6644
6645	/**
6646	* Sets an external interrupt as pending-for-injection into the VM.
6647	*
6648	* @param pVCpu The cross context virtual CPU structure.
6649	* @param u8Interrupt The external interrupt vector.
6650	*/
6651	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
6652	{
6653	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
6654	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6655	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6656	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6657	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6658	}
6659
6660
6661	/**
6662	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
6663	*
6664	* @param pVCpu The cross context virtual CPU structure.
6665	*/
6666	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
6667	{
6668	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
6669	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
6670	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6671	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6672	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6673	}
6674
6675
6676	/**
6677	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
6678	*
6679	* @param pVCpu The cross context virtual CPU structure.
6680	*/
6681	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
6682	{
6683	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
6684	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6685	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6686	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6687	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6688	}
6689
6690
6691	/**
6692	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
6693	*
6694	* @param pVCpu The cross context virtual CPU structure.
6695	*/
6696	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
6697	{
6698	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
6699	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6700	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6701	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6702	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6703	}
6704
6705
6706	/**
6707	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
6708	*
6709	* @param pVCpu The cross context virtual CPU structure.
6710	*/
6711	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
6712	{
6713	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
6714	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6715	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6716	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6717	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6718	}
6719
6720
6721	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6722	/**
6723	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
6724	*
6725	* @param pVCpu The cross context virtual CPU structure.
6726	* @param u32ErrCode The error code for the general-protection exception.
6727	*/
6728	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
6729	{
6730	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
6731	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6732	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6733	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6734	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6735	}
6736
6737
6738	/**
6739	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
6740	*
6741	* @param pVCpu The cross context virtual CPU structure.
6742	* @param u32ErrCode The error code for the stack exception.
6743	*/
6744	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
6745	{
6746	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
6747	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6748	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6749	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6750	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6751	}
6752
6753
6754	/**
6755	* Decodes the memory operand of an instruction that caused a VM-exit.
6756	*
6757	* The VM-exit qualification field provides the displacement field for memory
6758	* operand instructions, if any.
6759	*
6760	* @returns Strict VBox status code (i.e. informational status codes too).
6761	* @retval VINF_SUCCESS if the operand was successfully decoded.
6762	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
6763	* operand.
6764	* @param pVCpu The cross context virtual CPU structure.
6765	* @param uExitInstrInfo The VM-exit instruction information field.
6766	* @param enmMemAccess The memory operand's access type (read or write).
6767	* @param GCPtrDisp The instruction displacement field, if any. For
6768	* RIP-relative addressing pass RIP + displacement here.
6769	* @param pGCPtrMem Where to store the effective destination memory address.
6770	*/
6771	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
6772	PRTGCPTR pGCPtrMem)
6773	{
6774	Assert(pGCPtrMem);
6775	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
6776	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
6777	\| CPUMCTX_EXTRN_CR0);
6778
6779	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
6780	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
6781	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
6782
6783	VMXEXITINSTRINFO ExitInstrInfo;
6784	ExitInstrInfo.u = uExitInstrInfo;
6785	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
6786	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
6787	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
6788	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
6789	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
6790	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
6791	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
6792	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
6793	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
6794
6795	/*
6796	* Validate instruction information.
6797	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
6798	*/
6799	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
6800	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
6801	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
6802	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
6803	AssertLogRelMsgReturn(fIsMemOperand,
6804	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
6805
6806	/*
6807	* Compute the complete effective address.
6808	*
6809	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
6810	* See AMD spec. 4.5.2 "Segment Registers".
6811	*/
6812	RTGCPTR GCPtrMem = GCPtrDisp;
6813	if (fBaseRegValid)
6814	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
6815	if (fIdxRegValid)
6816	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
6817
6818	RTGCPTR const GCPtrOff = GCPtrMem;
6819	if ( !fIsLongMode
6820	\|\| iSegReg >= X86_SREG_FS)
6821	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
6822	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
6823
6824	/*
6825	* Validate effective address.
6826	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
6827	*/
6828	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
6829	Assert(cbAccess > 0);
6830	if (fIsLongMode)
6831	{
6832	if (X86_IS_CANONICAL(GCPtrMem))
6833	{
6834	*pGCPtrMem = GCPtrMem;
6835	return VINF_SUCCESS;
6836	}
6837
6838	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
6839	* "Data Limit Checks in 64-bit Mode". */
6840	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
6841	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6842	return VINF_HM_PENDING_XCPT;
6843	}
6844
6845	/*
6846	* This is a watered down version of iemMemApplySegment().
6847	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
6848	* and segment CPL/DPL checks are skipped.
6849	*/
6850	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
6851	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
6852	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
6853
6854	/* Check if the segment is present and usable. */
6855	if ( pSel->Attr.n.u1Present
6856	&& !pSel->Attr.n.u1Unusable)
6857	{
6858	Assert(pSel->Attr.n.u1DescType);
6859	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
6860	{
6861	/* Check permissions for the data segment. */
6862	if ( enmMemAccess == VMXMEMACCESS_WRITE
6863	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
6864	{
6865	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6866	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
6867	return VINF_HM_PENDING_XCPT;
6868	}
6869
6870	/* Check limits if it's a normal data segment. */
6871	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
6872	{
6873	if ( GCPtrFirst32 > pSel->u32Limit
6874	\|\| GCPtrLast32 > pSel->u32Limit)
6875	{
6876	Log4Func(("Data segment limit exceeded."
6877	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6878	GCPtrLast32, pSel->u32Limit));
6879	if (iSegReg == X86_SREG_SS)
6880	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6881	else
6882	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6883	return VINF_HM_PENDING_XCPT;
6884	}
6885	}
6886	else
6887	{
6888	/* Check limits if it's an expand-down data segment.
6889	Note! The upper boundary is defined by the B bit, not the G bit! */
6890	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
6891	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
6892	{
6893	Log4Func(("Expand-down data segment limit exceeded."
6894	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6895	GCPtrLast32, pSel->u32Limit));
6896	if (iSegReg == X86_SREG_SS)
6897	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6898	else
6899	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6900	return VINF_HM_PENDING_XCPT;
6901	}
6902	}
6903	}
6904	else
6905	{
6906	/* Check permissions for the code segment. */
6907	if ( enmMemAccess == VMXMEMACCESS_WRITE
6908	\|\| ( enmMemAccess == VMXMEMACCESS_READ
6909	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
6910	{
6911	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6912	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6913	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6914	return VINF_HM_PENDING_XCPT;
6915	}
6916
6917	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6918	if ( GCPtrFirst32 > pSel->u32Limit
6919	\|\| GCPtrLast32 > pSel->u32Limit)
6920	{
6921	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6922	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6923	if (iSegReg == X86_SREG_SS)
6924	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6925	else
6926	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6927	return VINF_HM_PENDING_XCPT;
6928	}
6929	}
6930	}
6931	else
6932	{
6933	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6934	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6935	return VINF_HM_PENDING_XCPT;
6936	}
6937
6938	*pGCPtrMem = GCPtrMem;
6939	return VINF_SUCCESS;
6940	}
6941
6942
6943	/**
6944	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6945	* guest attempting to execute a VMX instruction.
6946	*
6947	* @returns Strict VBox status code (i.e. informational status codes too).
6948	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
6949	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
6950	*
6951	* @param pVCpu The cross context virtual CPU structure.
6952	* @param uExitReason The VM-exit reason.
6953	*
6954	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
6955	* @remarks No-long-jump zone!!!
6956	*/
6957	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
6958	{
6959	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
6960	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
6961
6962	if ( CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx)
6963	\|\| ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6964	&& !CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)))
6965	{
6966	Log4Func(("In real/v86-mode or long-mode outside 64-bit code segment -> #UD\n"));
6967	hmR0VmxSetPendingXcptUD(pVCpu);
6968	return VINF_HM_PENDING_XCPT;
6969	}
6970
6971	if (uExitReason == VMX_EXIT_VMXON)
6972	{
6973	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
6974
6975	/*
6976	* We check CR4.VMXE because it is required to be always set while in VMX operation
6977	* by physical CPUs and our CR4 read shadow is only consulted when executing specific
6978	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
6979	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
6980	*/
6981	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
6982	{
6983	Log4Func(("CR4.VMXE is not set -> #UD\n"));
6984	hmR0VmxSetPendingXcptUD(pVCpu);
6985	return VINF_HM_PENDING_XCPT;
6986	}
6987	}
6988	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
6989	{
6990	/*
6991	* The guest has not entered VMX operation but attempted to execute a VMX instruction
6992	* (other than VMXON), we need to raise a #UD.
6993	*/
6994	Log4Func(("Not in VMX root mode -> #UD\n"));
6995	hmR0VmxSetPendingXcptUD(pVCpu);
6996	return VINF_HM_PENDING_XCPT;
6997	}
6998
6999	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
7000	{
7001	/*
7002	* The nested-guest attempted to execute a VMX instruction, cause a VM-exit and let
7003	* the guest hypervisor deal with it.
7004	*/
7005	/** @todo NSTVMX: Trigger a VM-exit */
7006	}
7007
7008	/*
7009	* VMX instructions require CPL 0 except in VMX non-root mode where the VM-exit intercept
7010	* (above) takes preceedence over the CPL check.
7011	*/
7012	if (CPUMGetGuestCPL(pVCpu) > 0)
7013	{
7014	Log4Func(("CPL > 0 -> #GP(0)\n"));
7015	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7016	return VINF_HM_PENDING_XCPT;
7017	}
7018
7019	return VINF_SUCCESS;
7020	}
7021	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7022
7023
7024	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7025	{
7026	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7027
7028	/*
7029	* If VT-x marks the segment as unusable, most other bits remain undefined:
7030	* - For CS the L, D and G bits have meaning.
7031	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7032	* - For the remaining data segments no bits are defined.
7033	*
7034	* The present bit and the unusable bit has been observed to be set at the
7035	* same time (the selector was supposed to be invalid as we started executing
7036	* a V8086 interrupt in ring-0).
7037	*
7038	* What should be important for the rest of the VBox code, is that the P bit is
7039	* cleared. Some of the other VBox code recognizes the unusable bit, but
7040	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7041	* safe side here, we'll strip off P and other bits we don't care about. If
7042	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7043	*
7044	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7045	*/
7046	#ifdef VBOX_STRICT
7047	uint32_t const uAttr = pSelReg->Attr.u;
7048	#endif
7049
7050	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7051	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7052	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7053
7054	#ifdef VBOX_STRICT
7055	VMMRZCallRing3Disable(pVCpu);
7056	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7057	# ifdef DEBUG_bird
7058	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7059	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7060	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7061	# endif
7062	NOREF(uAttr);
7063	VMMRZCallRing3Enable(pVCpu);
7064	#else
7065	RT_NOREF2(pVCpu, idxSel);
7066	#endif
7067	}
7068
7069
7070	/**
7071	* Imports a guest segment register from the current VMCS into the guest-CPU
7072	* context.
7073	*
7074	* @returns VBox status code.
7075	* @param pVCpu The cross context virtual CPU structure.
7076	* @param iSegReg The segment register number (X86_SREG_XXX).
7077	*
7078	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7079	* do not log!
7080	*/
7081	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7082	{
7083	Assert(iSegReg < X86_SREG_COUNT);
7084
7085	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7086	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7087	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7088	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
7089	uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
7090	#else
7091	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7092	#endif
7093	uint64_t u64Base;
7094	uint32_t u32Sel, u32Limit, u32Attr;
7095	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7096	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7097	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7098	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
7099	if (RT_SUCCESS(rc))
7100	{
7101	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7102	pSelReg->Sel = u32Sel;
7103	pSelReg->ValidSel = u32Sel;
7104	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7105	pSelReg->u32Limit = u32Limit;
7106	pSelReg->u64Base = u64Base;
7107	pSelReg->Attr.u = u32Attr;
7108	if (u32Attr & X86DESCATTR_UNUSABLE)
7109	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7110	}
7111	return rc;
7112	}
7113
7114
7115	/**
7116	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7117	*
7118	* @returns VBox status code.
7119	* @param pVCpu The cross context virtual CPU structure.
7120	*
7121	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7122	* do not log!
7123	*/
7124	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7125	{
7126	uint64_t u64Base;
7127	uint32_t u32Sel, u32Limit, u32Attr;
7128	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7129	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7130	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7131	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7132
7133	if (RT_SUCCESS(rc))
7134	{
7135	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7136	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7137	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7138	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7139	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7140	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7141	if (u32Attr & X86DESCATTR_UNUSABLE)
7142	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7143	}
7144	return rc;
7145	}
7146
7147
7148	/**
7149	* Imports the guest TR from the current VMCS into the guest-CPU context.
7150	*
7151	* @returns VBox status code.
7152	* @param pVCpu The cross context virtual CPU structure.
7153	*
7154	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7155	* do not log!
7156	*/
7157	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7158	{
7159	uint32_t u32Sel, u32Limit, u32Attr;
7160	uint64_t u64Base;
7161	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7162	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7163	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7164	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7165	AssertRCReturn(rc, rc);
7166
7167	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7168	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7169	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7170	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7171	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7172	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7173	/* TR is the only selector that can never be unusable. */
7174	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7175	return VINF_SUCCESS;
7176	}
7177
7178
7179	/**
7180	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7181	*
7182	* @returns VBox status code.
7183	* @param pVCpu The cross context virtual CPU structure.
7184	*
7185	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7186	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7187	* instead!!!
7188	*/
7189	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7190	{
7191	uint64_t u64Val;
7192	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7193	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7194	{
7195	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7196	if (RT_SUCCESS(rc))
7197	{
7198	pCtx->rip = u64Val;
7199	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7200	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7201	}
7202	return rc;
7203	}
7204	return VINF_SUCCESS;
7205	}
7206
7207
7208	/**
7209	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7210	*
7211	* @returns VBox status code.
7212	* @param pVCpu The cross context virtual CPU structure.
7213	* @param pVmcsInfo The VMCS info. object.
7214	*
7215	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7216	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7217	* instead!!!
7218	*/
7219	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7220	{
7221	uint32_t u32Val;
7222	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7223	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7224	{
7225	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7226	if (RT_SUCCESS(rc))
7227	{
7228	pCtx->eflags.u32 = u32Val;
7229
7230	/* Restore eflags for real-on-v86-mode hack. */
7231	if (pVmcsInfo->RealMode.fRealOnV86Active)
7232	{
7233	pCtx->eflags.Bits.u1VM = 0;
7234	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7235	}
7236	}
7237	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7238	return rc;
7239	}
7240	return VINF_SUCCESS;
7241	}
7242
7243
7244	/**
7245	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7246	* context.
7247	*
7248	* @returns VBox status code.
7249	* @param pVCpu The cross context virtual CPU structure.
7250	* @param pVmcsInfo The VMCS info. object.
7251	*
7252	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7253	* do not log!
7254	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7255	* instead!!!
7256	*/
7257	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7258	{
7259	uint32_t u32Val;
7260	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7261	if (RT_SUCCESS(rc))
7262	{
7263	if (!u32Val)
7264	{
7265	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7266	{
7267	rc = hmR0VmxImportGuestRip(pVCpu);
7268	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7269	AssertRCReturn(rc, rc);
7270	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7271	}
7272
7273	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7274	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7275	}
7276	else
7277	{
7278	/*
7279	* We must import RIP here to set our EM interrupt-inhibited state.
7280	* We also import RFLAGS as our code that evaluates pending interrupts
7281	* before VM-entry requires it.
7282	*/
7283	rc = hmR0VmxImportGuestRip(pVCpu);
7284	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7285	if (RT_SUCCESS(rc))
7286	{
7287	if (u32Val & ( VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
7288	\| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7289	{
7290	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7291	}
7292	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7293	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7294
7295	if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
7296	{
7297	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7298	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
7299	}
7300	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7301	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7302	}
7303	}
7304	}
7305	return rc;
7306	}
7307
7308
7309	/**
7310	* Worker for VMXR0ImportStateOnDemand.
7311	*
7312	* @returns VBox status code.
7313	* @param pVCpu The cross context virtual CPU structure.
7314	* @param pVmcsInfo The VMCS info. object.
7315	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7316	*/
7317	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7318	{
7319	#define VMXLOCAL_BREAK_RC(a_rc) \
7320	if (RT_SUCCESS(a_rc)) \
7321	{ } \
7322	else \
7323	break
7324
7325	int rc = VINF_SUCCESS;
7326	PVM pVM = pVCpu->CTX_SUFF(pVM);
7327	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7328	uint64_t u64Val;
7329	uint32_t u32Val;
7330
7331	STAM_PROFILE_ADV_START(& pVCpu->hm.s.StatImportGuestState, x);
7332
7333	/*
7334	* We disable interrupts to make the updating of the state and in particular
7335	* the fExtrn modification atomic wrt to preemption hooks.
7336	*/
7337	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7338
7339	fWhat &= pCtx->fExtrn;
7340	if (fWhat)
7341	{
7342	do
7343	{
7344	if (fWhat & CPUMCTX_EXTRN_RIP)
7345	{
7346	rc = hmR0VmxImportGuestRip(pVCpu);
7347	VMXLOCAL_BREAK_RC(rc);
7348	}
7349
7350	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7351	{
7352	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7353	VMXLOCAL_BREAK_RC(rc);
7354	}
7355
7356	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7357	{
7358	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7359	VMXLOCAL_BREAK_RC(rc);
7360	}
7361
7362	if (fWhat & CPUMCTX_EXTRN_RSP)
7363	{
7364	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
7365	VMXLOCAL_BREAK_RC(rc);
7366	pCtx->rsp = u64Val;
7367	}
7368
7369	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7370	{
7371	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7372	if (fWhat & CPUMCTX_EXTRN_CS)
7373	{
7374	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7375	rc \|= hmR0VmxImportGuestRip(pVCpu);
7376	if (fRealOnV86Active)
7377	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7378	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7379	}
7380	if (fWhat & CPUMCTX_EXTRN_SS)
7381	{
7382	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7383	if (fRealOnV86Active)
7384	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7385	}
7386	if (fWhat & CPUMCTX_EXTRN_DS)
7387	{
7388	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7389	if (fRealOnV86Active)
7390	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7391	}
7392	if (fWhat & CPUMCTX_EXTRN_ES)
7393	{
7394	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7395	if (fRealOnV86Active)
7396	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7397	}
7398	if (fWhat & CPUMCTX_EXTRN_FS)
7399	{
7400	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7401	if (fRealOnV86Active)
7402	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7403	}
7404	if (fWhat & CPUMCTX_EXTRN_GS)
7405	{
7406	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7407	if (fRealOnV86Active)
7408	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7409	}
7410	VMXLOCAL_BREAK_RC(rc);
7411	}
7412
7413	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7414	{
7415	if (fWhat & CPUMCTX_EXTRN_LDTR)
7416	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
7417
7418	if (fWhat & CPUMCTX_EXTRN_GDTR)
7419	{
7420	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
7421	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
7422	pCtx->gdtr.pGdt = u64Val;
7423	pCtx->gdtr.cbGdt = u32Val;
7424	}
7425
7426	/* Guest IDTR. */
7427	if (fWhat & CPUMCTX_EXTRN_IDTR)
7428	{
7429	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
7430	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
7431	pCtx->idtr.pIdt = u64Val;
7432	pCtx->idtr.cbIdt = u32Val;
7433	}
7434
7435	/* Guest TR. */
7436	if (fWhat & CPUMCTX_EXTRN_TR)
7437	{
7438	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7439	don't need to import that one. */
7440	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7441	rc \|= hmR0VmxImportGuestTr(pVCpu);
7442	}
7443	VMXLOCAL_BREAK_RC(rc);
7444	}
7445
7446	if (fWhat & CPUMCTX_EXTRN_DR7)
7447	{
7448	if (!pVCpu->hm.s.fUsingHyperDR7)
7449	{
7450	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
7451	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
7452	VMXLOCAL_BREAK_RC(rc);
7453	pCtx->dr[7] = u32Val;
7454	}
7455	}
7456
7457	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7458	{
7459	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
7460	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
7461	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
7462	pCtx->SysEnter.cs = u32Val;
7463	VMXLOCAL_BREAK_RC(rc);
7464	}
7465
7466	#if HC_ARCH_BITS == 64
7467	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7468	{
7469	if ( pVM->hm.s.fAllow64BitGuests
7470	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7471	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7472	}
7473
7474	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7475	{
7476	if ( pVM->hm.s.fAllow64BitGuests
7477	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7478	{
7479	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7480	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7481	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7482	}
7483	}
7484	#endif
7485
7486	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7487	#if HC_ARCH_BITS == 32
7488	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
7489	#endif
7490	)
7491	{
7492	PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7493	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7494	Assert(cMsrs == 0 \|\| pMsr != NULL);
7495	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
7496	{
7497	switch (pMsr->u32Msr)
7498	{
7499	#if HC_ARCH_BITS == 32
7500	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
7501	case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
7502	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
7503	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
7504	#endif
7505	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
7506	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
7507	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7508
7509	default:
7510	{
7511	pVCpu->hm.s.u32HMError = pMsr->u32Msr;
7512	ASMSetFlags(fEFlags);
7513	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
7514	cMsrs));
7515	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7516	}
7517	}
7518	}
7519	}
7520
7521	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7522	{
7523	uint64_t u64Shadow;
7524	if (fWhat & CPUMCTX_EXTRN_CR0)
7525	{
7526	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7527	* remove when we drop 32-bit host w/ 64-bit host support, see
7528	* @bugref{9180#c39}. */
7529	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7530	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
7531	VMXLOCAL_BREAK_RC(rc);
7532	u64Val = u32Val;
7533	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7534	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7535	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7536	CPUMSetGuestCR0(pVCpu, u64Val);
7537	VMMRZCallRing3Enable(pVCpu);
7538	}
7539
7540	if (fWhat & CPUMCTX_EXTRN_CR4)
7541	{
7542	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7543	* remove when we drop 32-bit host w/ 64-bit host support, see
7544	* @bugref{9180#c39}. */
7545	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7546	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
7547	VMXLOCAL_BREAK_RC(rc);
7548	u64Val = u32Val;
7549	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
7550	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7551	pCtx->cr4 = u64Val;
7552	}
7553
7554	if (fWhat & CPUMCTX_EXTRN_CR3)
7555	{
7556	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7557	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7558	\|\| ( pVM->hm.s.fNestedPaging
7559	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7560	{
7561	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
7562	if (pCtx->cr3 != u64Val)
7563	{
7564	pCtx->cr3 = u64Val;
7565	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7566	}
7567
7568	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7569	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7570	if (CPUMIsGuestInPAEModeEx(pCtx))
7571	{
7572	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
7573	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
7574	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
7575	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
7576	VMXLOCAL_BREAK_RC(rc);
7577	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7578	}
7579	}
7580	}
7581	}
7582	} while (0);
7583
7584	if (RT_SUCCESS(rc))
7585	{
7586	/* Update fExtrn. */
7587	pCtx->fExtrn &= ~fWhat;
7588
7589	/* If everything has been imported, clear the HM keeper bit. */
7590	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7591	{
7592	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7593	Assert(!pCtx->fExtrn);
7594	}
7595	}
7596	}
7597	else
7598	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7599
7600	ASMSetFlags(fEFlags);
7601
7602	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7603
7604	if (RT_SUCCESS(rc))
7605	{ /* likely */ }
7606	else
7607	return rc;
7608
7609	/*
7610	* Honor any pending CR3 updates.
7611	*
7612	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
7613	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7614	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7615	*
7616	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7617	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7618	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7619	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7620	*
7621	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7622	*/
7623	if (VMMRZCallRing3IsEnabled(pVCpu))
7624	{
7625	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7626	{
7627	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7628	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7629	}
7630
7631	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7632	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7633
7634	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7635	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7636	}
7637
7638	return VINF_SUCCESS;
7639	#undef VMXLOCAL_BREAK_RC
7640	}
7641
7642
7643	/**
7644	* Saves the guest state from the VMCS into the guest-CPU context.
7645	*
7646	* @returns VBox status code.
7647	* @param pVCpu The cross context virtual CPU structure.
7648	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7649	*/
7650	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
7651	{
7652	PCVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7653	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7654	}
7655
7656
7657	/**
7658	* Check per-VM and per-VCPU force flag actions that require us to go back to
7659	* ring-3 for one reason or another.
7660	*
7661	* @returns Strict VBox status code (i.e. informational status codes too)
7662	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7663	* ring-3.
7664	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7665	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7666	* interrupts)
7667	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7668	* all EMTs to be in ring-3.
7669	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7670	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7671	* to the EM loop.
7672	*
7673	* @param pVCpu The cross context virtual CPU structure.
7674	* @param fStepping Whether we are single-stepping the guest using the
7675	* hypervisor debugger.
7676	*/
7677	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
7678	{
7679	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7680
7681	/*
7682	* Update pending interrupts into the APIC's IRR.
7683	*/
7684	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7685	APICUpdatePendingInterrupts(pVCpu);
7686
7687	/*
7688	* Anything pending? Should be more likely than not if we're doing a good job.
7689	*/
7690	PVM pVM = pVCpu->CTX_SUFF(pVM);
7691	if ( !fStepping
7692	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7693	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7694	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7695	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7696	return VINF_SUCCESS;
7697
7698	/* Pending PGM C3 sync. */
7699	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7700	{
7701	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7702	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7703	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7704	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7705	if (rcStrict2 != VINF_SUCCESS)
7706	{
7707	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7708	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7709	return rcStrict2;
7710	}
7711	}
7712
7713	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7714	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7715	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7716	{
7717	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7718	int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7719	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7720	return rc2;
7721	}
7722
7723	/* Pending VM request packets, such as hardware interrupts. */
7724	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7725	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7726	{
7727	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7728	return VINF_EM_PENDING_REQUEST;
7729	}
7730
7731	/* Pending PGM pool flushes. */
7732	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7733	{
7734	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7735	return VINF_PGM_POOL_FLUSH_PENDING;
7736	}
7737
7738	/* Pending DMA requests. */
7739	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7740	{
7741	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7742	return VINF_EM_RAW_TO_R3;
7743	}
7744
7745	return VINF_SUCCESS;
7746	}
7747
7748
7749	/**
7750	* Converts any TRPM trap into a pending HM event. This is typically used when
7751	* entering from ring-3 (not longjmp returns).
7752	*
7753	* @param pVCpu The cross context virtual CPU structure.
7754	*/
7755	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7756	{
7757	Assert(TRPMHasTrap(pVCpu));
7758	Assert(!pVCpu->hm.s.Event.fPending);
7759
7760	uint8_t uVector;
7761	TRPMEVENT enmTrpmEvent;
7762	RTGCUINT uErrCode;
7763	RTGCUINTPTR GCPtrFaultAddress;
7764	uint8_t cbInstr;
7765
7766	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7767	AssertRC(rc);
7768
7769	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7770	uint32_t u32IntInfo = uVector \| VMX_EXIT_INT_INFO_VALID;
7771	if (enmTrpmEvent == TRPM_TRAP)
7772	{
7773	/** @todo r=ramshankar: TRPM currently offers no way to determine a \#DB that was
7774	* generated using INT1 (ICEBP). */
7775	switch (uVector)
7776	{
7777	case X86_XCPT_NMI:
7778	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7779	break;
7780
7781	case X86_XCPT_BP:
7782	case X86_XCPT_OF:
7783	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7784	break;
7785
7786	case X86_XCPT_PF:
7787	case X86_XCPT_DF:
7788	case X86_XCPT_TS:
7789	case X86_XCPT_NP:
7790	case X86_XCPT_SS:
7791	case X86_XCPT_GP:
7792	case X86_XCPT_AC:
7793	u32IntInfo \|= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7794	RT_FALL_THRU();
7795	default:
7796	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7797	break;
7798	}
7799	}
7800	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7801	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7802	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7803	{
7804	switch (uVector)
7805	{
7806	case X86_XCPT_BP:
7807	case X86_XCPT_OF:
7808	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7809	break;
7810
7811	default:
7812	Assert(uVector == X86_XCPT_DB);
7813	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7814	break;
7815	}
7816	}
7817	else
7818	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7819
7820	rc = TRPMResetTrap(pVCpu);
7821	AssertRC(rc);
7822	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7823	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7824
7825	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7826	}
7827
7828
7829	/**
7830	* Converts the pending HM event into a TRPM trap.
7831	*
7832	* @param pVCpu The cross context virtual CPU structure.
7833	*/
7834	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7835	{
7836	Assert(pVCpu->hm.s.Event.fPending);
7837
7838	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7839	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7840	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
7841	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7842
7843	/* If a trap was already pending, we did something wrong! */
7844	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7845
7846	/** @todo Use HMVmxEventToTrpmEventType() later. */
7847	TRPMEVENT enmTrapType;
7848	switch (uVectorType)
7849	{
7850	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7851	enmTrapType = TRPM_HARDWARE_INT;
7852	break;
7853
7854	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7855	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7856	enmTrapType = TRPM_TRAP;
7857	break;
7858
7859	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT: /* #DB (INT1/ICEBP). */
7860	Assert(uVector == X86_XCPT_DB);
7861	enmTrapType = TRPM_SOFTWARE_INT;
7862	break;
7863
7864	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP (INT3) and #OF (INTO) */
7865	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7866	enmTrapType = TRPM_SOFTWARE_INT;
7867	break;
7868
7869	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7870	enmTrapType = TRPM_SOFTWARE_INT;
7871	break;
7872
7873	default:
7874	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7875	enmTrapType = TRPM_32BIT_HACK;
7876	break;
7877	}
7878
7879	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7880
7881	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7882	AssertRC(rc);
7883
7884	if (fErrorCodeValid)
7885	TRPMSetErrorCode(pVCpu, uErrorCode);
7886
7887	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7888	&& uVector == X86_XCPT_PF)
7889	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7890	else if (enmTrapType == TRPM_SOFTWARE_INT)
7891	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7892
7893	/* We're now done converting the pending event. */
7894	pVCpu->hm.s.Event.fPending = false;
7895	}
7896
7897
7898	/**
7899	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7900	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7901	*
7902	* @param pVCpu The cross context virtual CPU structure.
7903	* @param pVmcsInfo The VMCS info. object.
7904	*/
7905	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
7906	{
7907	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7908	{
7909	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7910	{
7911	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7912	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7913	AssertRC(rc);
7914	}
7915	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
7916	}
7917
7918
7919	/**
7920	* Clears the interrupt-window exiting control in the VMCS.
7921	*
7922	* @param pVmcsInfo The VMCS info. object.
7923	*/
7924	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7925	{
7926	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7927	{
7928	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7929	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7930	}
7931	return VINF_SUCCESS;
7932	}
7933
7934
7935	/**
7936	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7937	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7938	*
7939	* @param pVCpu The cross context virtual CPU structure.
7940	* @param pVmcsInfo The VMCS info. object.
7941	*/
7942	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
7943	{
7944	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7945	{
7946	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7947	{
7948	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7949	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7950	AssertRC(rc);
7951	Log4Func(("Setup NMI-window exiting\n"));
7952	}
7953	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7954	}
7955
7956
7957	/**
7958	* Clears the NMI-window exiting control in the VMCS.
7959	*
7960	* @param pVmcsInfo The VMCS info. object.
7961	*/
7962	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7963	{
7964	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7965	{
7966	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7967	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7968	}
7969	return VINF_SUCCESS;
7970	}
7971
7972
7973	/**
7974	* Does the necessary state syncing before returning to ring-3 for any reason
7975	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7976	*
7977	* @returns VBox status code.
7978	* @param pVCpu The cross context virtual CPU structure.
7979	* @param fImportState Whether to import the guest state from the VMCS back
7980	* to the guest-CPU context.
7981	*
7982	* @remarks No-long-jmp zone!!!
7983	*/
7984	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
7985	{
7986	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7987	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7988
7989	RTCPUID idCpu = RTMpCpuId();
7990	Log4Func(("HostCpuId=%u\n", idCpu));
7991
7992	/*
7993	* !!! IMPORTANT !!!
7994	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7995	*/
7996
7997	/* Save the guest state if necessary. */
7998	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7999	if (fImportState)
8000	{
8001	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8002	AssertRCReturn(rc, rc);
8003	}
8004
8005	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8006	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8007	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8008
8009	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8010	#ifdef VBOX_STRICT
8011	if (CPUMIsHyperDebugStateActive(pVCpu))
8012	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8013	#endif
8014	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8015	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8016	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8017
8018	#if HC_ARCH_BITS == 64
8019	/* Restore host-state bits that VT-x only restores partially. */
8020	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8021	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8022	{
8023	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8024	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8025	}
8026	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8027	#endif
8028
8029	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8030	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8031	{
8032	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8033	if (!fImportState)
8034	{
8035	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8036	AssertRCReturn(rc, rc);
8037	}
8038	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8039	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8040	}
8041	else
8042	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8043
8044	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8045	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8046
8047	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8048	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8049	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8050	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8051	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8052	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8053	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8054	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8055	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8056
8057	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8058
8059	/** @todo This partially defeats the purpose of having preemption hooks.
8060	* The problem is, deregistering the hooks should be moved to a place that
8061	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8062	* context.
8063	*/
8064	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8065	AssertRCReturn(rc, rc);
8066
8067	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8068	NOREF(idCpu);
8069	return VINF_SUCCESS;
8070	}
8071
8072
8073	/**
8074	* Leaves the VT-x session.
8075	*
8076	* @returns VBox status code.
8077	* @param pVCpu The cross context virtual CPU structure.
8078	*
8079	* @remarks No-long-jmp zone!!!
8080	*/
8081	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8082	{
8083	HM_DISABLE_PREEMPT(pVCpu);
8084	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8085	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8086	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8087
8088	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8089	and done this from the VMXR0ThreadCtxCallback(). */
8090	if (!pVCpu->hm.s.fLeaveDone)
8091	{
8092	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8093	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8094	pVCpu->hm.s.fLeaveDone = true;
8095	}
8096	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8097
8098	/*
8099	* !!! IMPORTANT !!!
8100	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8101	*/
8102
8103	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8104	/** @todo Deregistering here means we need to VMCLEAR always
8105	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8106	* for calling VMMR0ThreadCtxHookDisable here! */
8107	VMMR0ThreadCtxHookDisable(pVCpu);
8108
8109	/* Leave HM context. This takes care of local init (term). */
8110	int rc = HMR0LeaveCpu(pVCpu);
8111
8112	HM_RESTORE_PREEMPT();
8113	return rc;
8114	}
8115
8116
8117	/**
8118	* Does the necessary state syncing before doing a longjmp to ring-3.
8119	*
8120	* @returns VBox status code.
8121	* @param pVCpu The cross context virtual CPU structure.
8122	*
8123	* @remarks No-long-jmp zone!!!
8124	*/
8125	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8126	{
8127	return hmR0VmxLeaveSession(pVCpu);
8128	}
8129
8130
8131	/**
8132	* Take necessary actions before going back to ring-3.
8133	*
8134	* An action requires us to go back to ring-3. This function does the necessary
8135	* steps before we can safely return to ring-3. This is not the same as longjmps
8136	* to ring-3, this is voluntary and prepares the guest so it may continue
8137	* executing outside HM (recompiler/IEM).
8138	*
8139	* @returns VBox status code.
8140	* @param pVCpu The cross context virtual CPU structure.
8141	* @param rcExit The reason for exiting to ring-3. Can be
8142	* VINF_VMM_UNKNOWN_RING3_CALL.
8143	*/
8144	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8145	{
8146	Assert(pVCpu);
8147	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8148
8149	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8150	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8151	{
8152	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8153	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8154	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8155	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8156	}
8157
8158	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8159	VMMRZCallRing3Disable(pVCpu);
8160	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8161
8162	/*
8163	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8164	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8165	*
8166	* This is because execution may continue from ring-3 and we would need to inject
8167	* the event from there (hence place it back in TRPM).
8168	*/
8169	if (pVCpu->hm.s.Event.fPending)
8170	{
8171	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8172	Assert(!pVCpu->hm.s.Event.fPending);
8173
8174	/* Clear the events from the VMCS. */
8175	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8176	AssertRCReturn(rc, rc);
8177	}
8178	#ifdef VBOX_STRICT
8179	else
8180	{
8181	/*
8182	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8183	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8184	* occasionally, see @bugref{9180#c42}.
8185	*/
8186	uint32_t uEntryIntInfo;
8187	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8188	AssertRC(rc);
8189	Assert(!VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8190	}
8191	#endif
8192
8193	/*
8194	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8195	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8196	* (e.g. TPR below threshold).
8197	*/
8198	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8199	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8200	AssertRCReturn(rc, rc);
8201
8202	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8203	and if we're injecting an event we should have a TRPM trap pending. */
8204	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8205	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8206	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8207	#endif
8208
8209	/* Save guest state and restore host state bits. */
8210	rc = hmR0VmxLeaveSession(pVCpu);
8211	AssertRCReturn(rc, rc);
8212	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8213
8214	/* Thread-context hooks are unregistered at this point!!! */
8215
8216	/* Sync recompiler state. */
8217	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8218	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8219	\| CPUM_CHANGED_LDTR
8220	\| CPUM_CHANGED_GDTR
8221	\| CPUM_CHANGED_IDTR
8222	\| CPUM_CHANGED_TR
8223	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8224	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8225	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8226	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8227
8228	Assert(!pVCpu->hm.s.fClearTrapFlag);
8229
8230	/* Update the exit-to-ring 3 reason. */
8231	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8232
8233	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8234	if ( rcExit != VINF_EM_RAW_INTERRUPT
8235	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8236	{
8237	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8238	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8239	}
8240
8241	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8242
8243	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
8244	VMMRZCallRing3RemoveNotification(pVCpu);
8245	VMMRZCallRing3Enable(pVCpu);
8246
8247	return rc;
8248	}
8249
8250
8251	/**
8252	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8253	* longjump to ring-3 and possibly get preempted.
8254	*
8255	* @returns VBox status code.
8256	* @param pVCpu The cross context virtual CPU structure.
8257	* @param enmOperation The operation causing the ring-3 longjump.
8258	* @param pvUser User argument, currently unused, NULL.
8259	*/
8260	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
8261	{
8262	RT_NOREF(pvUser);
8263	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8264	{
8265	/*
8266	* !!! IMPORTANT !!!
8267	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8268	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8269	*/
8270	VMMRZCallRing3RemoveNotification(pVCpu);
8271	VMMRZCallRing3Disable(pVCpu);
8272	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
8273	RTThreadPreemptDisable(&PreemptState);
8274
8275	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8276	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8277	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8278	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8279
8280	#if HC_ARCH_BITS == 64
8281	/* Restore host-state bits that VT-x only restores partially. */
8282	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8283	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8284	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8285	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8286	#endif
8287
8288	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8289	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8290	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8291
8292	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8293	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8294	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8295
8296	/* Clear the current VMCS data back to memory. */
8297	hmR0VmxClearVmcs(pVmcsInfo);
8298
8299	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8300	VMMR0ThreadCtxHookDisable(pVCpu);
8301	HMR0LeaveCpu(pVCpu);
8302	RTThreadPreemptRestore(&PreemptState);
8303	return VINF_SUCCESS;
8304	}
8305
8306	Assert(pVCpu);
8307	Assert(pvUser);
8308	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8309	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8310
8311	VMMRZCallRing3Disable(pVCpu);
8312	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8313
8314	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8315
8316	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8317	AssertRCReturn(rc, rc);
8318
8319	VMMRZCallRing3Enable(pVCpu);
8320	return VINF_SUCCESS;
8321	}
8322
8323
8324	/**
8325	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8326	* stack.
8327	*
8328	* @returns Strict VBox status code (i.e. informational status codes too).
8329	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8330	* @param pVCpu The cross context virtual CPU structure.
8331	* @param uValue The value to push to the guest stack.
8332	*/
8333	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
8334	{
8335	/*
8336	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8337	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8338	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8339	*/
8340	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8341	if (pCtx->sp == 1)
8342	return VINF_EM_RESET;
8343	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8344	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8345	AssertRC(rc);
8346	return rc;
8347	}
8348
8349
8350	/**
8351	* Injects an event into the guest upon VM-entry by updating the relevant fields
8352	* in the VM-entry area in the VMCS.
8353	*
8354	* @returns Strict VBox status code (i.e. informational status codes too).
8355	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8356	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8357	*
8358	* @param pVCpu The cross context virtual CPU structure.
8359	* @param pVmxTransient The VMX-transient structure.
8360	* @param pEvent The event being injected.
8361	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state.
8362	* This will be updated if necessary. This cannot not
8363	* be NULL.
8364	* @param fStepping Whether we're single-stepping guest execution and
8365	* should return VINF_EM_DBG_STEPPED if the event is
8366	* injected directly (registers modified by us, not by
8367	* hardware on VM-entry).
8368	*/
8369	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8370	uint32_t *pfIntrState)
8371	{
8372	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8373	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8374	Assert(pfIntrState);
8375
8376	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8377	uint32_t u32IntInfo = pEvent->u64IntInfo;
8378	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8379	uint32_t const cbInstr = pEvent->cbInstr;
8380	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8381	uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8382	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8383
8384	#ifdef VBOX_STRICT
8385	/*
8386	* Validate the error-code-valid bit for hardware exceptions.
8387	* No error codes for exceptions in real-mode.
8388	*
8389	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8390	*/
8391	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8392	&& !CPUMIsGuestInRealModeEx(pCtx))
8393	{
8394	switch (uVector)
8395	{
8396	case X86_XCPT_PF:
8397	case X86_XCPT_DF:
8398	case X86_XCPT_TS:
8399	case X86_XCPT_NP:
8400	case X86_XCPT_SS:
8401	case X86_XCPT_GP:
8402	case X86_XCPT_AC:
8403	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8404	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8405	RT_FALL_THRU();
8406	default:
8407	break;
8408	}
8409	}
8410
8411	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8412	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8413	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8414	#endif
8415
8416	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8417
8418	/*
8419	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8420	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8421	* interrupt handler in the (real-mode) guest.
8422	*
8423	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8424	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8425	*/
8426	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8427	{
8428	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8429	{
8430	/*
8431	* For CPUs with unrestricted guest execution enabled and with the guest
8432	* in real-mode, we must not set the deliver-error-code bit.
8433	*
8434	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8435	*/
8436	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8437	}
8438	else
8439	{
8440	PVM pVM = pVCpu->CTX_SUFF(pVM);
8441	Assert(PDMVmmDevHeapIsEnabled(pVM));
8442	Assert(pVM->hm.s.vmx.pRealModeTSS);
8443	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8444
8445	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8446	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8447	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8448	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8449	AssertRCReturn(rc2, rc2);
8450
8451	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8452	size_t const cbIdtEntry = sizeof(X86IDTR16);
8453	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8454	{
8455	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8456	if (uVector == X86_XCPT_DF)
8457	return VINF_EM_RESET;
8458
8459	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8460	No error codes for exceptions in real-mode. */
8461	if (uVector == X86_XCPT_GP)
8462	{
8463	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8464	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8465	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8466	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8467	HMEVENT EventXcptDf;
8468	RT_ZERO(EventXcptDf);
8469	EventXcptDf.u64IntInfo = uXcptDfInfo;
8470	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8471	}
8472
8473	/*
8474	* If we're injecting an event with no valid IDT entry, inject a #GP.
8475	* No error codes for exceptions in real-mode.
8476	*
8477	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8478	*/
8479	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8480	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8481	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8482	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8483	HMEVENT EventXcptGp;
8484	RT_ZERO(EventXcptGp);
8485	EventXcptGp.u64IntInfo = uXcptGpInfo;
8486	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8487	}
8488
8489	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8490	uint16_t uGuestIp = pCtx->ip;
8491	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8492	{
8493	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8494	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8495	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8496	}
8497	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8498	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8499
8500	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8501	X86IDTR16 IdtEntry;
8502	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8503	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8504	AssertRCReturn(rc2, rc2);
8505
8506	/* Construct the stack frame for the interrupt/exception handler. */
8507	VBOXSTRICTRC rcStrict;
8508	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8509	if (rcStrict == VINF_SUCCESS)
8510	{
8511	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8512	if (rcStrict == VINF_SUCCESS)
8513	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8514	}
8515
8516	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8517	if (rcStrict == VINF_SUCCESS)
8518	{
8519	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8520	pCtx->rip = IdtEntry.offSel;
8521	pCtx->cs.Sel = IdtEntry.uSel;
8522	pCtx->cs.ValidSel = IdtEntry.uSel;
8523	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8524	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8525	&& uVector == X86_XCPT_PF)
8526	pCtx->cr2 = GCPtrFault;
8527
8528	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8529	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8530	\| HM_CHANGED_GUEST_RSP);
8531
8532	/*
8533	* If we delivered a hardware exception (other than an NMI) and if there was
8534	* block-by-STI in effect, we should clear it.
8535	*/
8536	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8537	{
8538	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8539	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8540	Log4Func(("Clearing inhibition due to STI\n"));
8541	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8542	}
8543
8544	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8545	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8546
8547	/*
8548	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8549	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8550	*/
8551	pVCpu->hm.s.Event.fPending = false;
8552
8553	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8554	if (fStepping)
8555	rcStrict = VINF_EM_DBG_STEPPED;
8556	}
8557	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8558	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8559	return rcStrict;
8560	}
8561	}
8562
8563	/*
8564	* Validate.
8565	*/
8566	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8567	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8568
8569	/*
8570	* Inject the event into the VMCS.
8571	*/
8572	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8573	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8574	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8575	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8576	AssertRCReturn(rc, rc);
8577
8578	/*
8579	* Update guest CR2 if this is a page-fault.
8580	*/
8581	if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8582	&& uVector == X86_XCPT_PF)
8583	pCtx->cr2 = GCPtrFault;
8584
8585	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8586	return VINF_SUCCESS;
8587	}
8588
8589
8590	/**
8591	* Evaluates the event to be delivered to the guest and sets it as the pending
8592	* event.
8593	*
8594	* @returns Strict VBox status code (i.e. informational status codes too).
8595	* @param pVCpu The cross context virtual CPU structure.
8596	* @param pVmxTransient The VMX-transient structure.
8597	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8598	*/
8599	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8600	{
8601	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8602	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8603
8604	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
8605	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmcsInfo);
8606	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8607	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8608	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8609
8610	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8611	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8612	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8613	Assert(!TRPMHasTrap(pVCpu));
8614	Assert(pfIntrState);
8615
8616	*pfIntrState = fIntrState;
8617
8618	/*
8619	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
8620	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
8621	*/
8622	/** @todo SMI. SMIs take priority over NMIs. */
8623	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
8624	{
8625	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8626	if ( !pVCpu->hm.s.Event.fPending
8627	&& !fBlockNmi
8628	&& !fBlockSti
8629	&& !fBlockMovSS)
8630	{
8631	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8632	if ( pVmxTransient->fIsNestedGuest
8633	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8634	return IEMExecVmxVmexitNmi(pVCpu);
8635	#endif
8636	hmR0VmxSetPendingXcptNmi(pVCpu);
8637	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8638	Log4Func(("Pending NMI\n"));
8639	}
8640	else
8641	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8642	}
8643	/*
8644	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
8645	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
8646	*/
8647	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8648	&& !pVCpu->hm.s.fSingleInstruction)
8649	{
8650	Assert(!DBGFIsStepping(pVCpu));
8651	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8652	AssertRCReturn(rc, rc);
8653	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8654	if ( !pVCpu->hm.s.Event.fPending
8655	&& !fBlockInt
8656	&& !fBlockSti
8657	&& !fBlockMovSS)
8658	{
8659	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8660	if ( pVmxTransient->fIsNestedGuest
8661	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
8662	{
8663	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector /, true / fIntPending */);
8664	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8665	return rcStrict;
8666	}
8667	#endif
8668	uint8_t u8Interrupt;
8669	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8670	if (RT_SUCCESS(rc))
8671	{
8672	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8673	if ( pVmxTransient->fIsNestedGuest
8674	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8675	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8676	{
8677	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8678	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8679	return rcStrict;
8680	}
8681	#endif
8682	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8683	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
8684	}
8685	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8686	{
8687	if ( !pVmxTransient->fIsNestedGuest
8688	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8689	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
8690	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8691
8692	/*
8693	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8694	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8695	* need to re-set this force-flag here.
8696	*/
8697	}
8698	else
8699	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8700	}
8701	else
8702	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8703	}
8704
8705	return VINF_SUCCESS;
8706	}
8707
8708
8709	/**
8710	* Injects any pending events into the guest if the guest is in a state to
8711	* receive them.
8712	*
8713	* @returns Strict VBox status code (i.e. informational status codes too).
8714	* @param pVCpu The cross context virtual CPU structure.
8715	* @param pVmxTransient The VMX-transient structure.
8716	* @param fIntrState The VT-x guest-interruptibility state.
8717	* @param fStepping Whether we are single-stepping the guest using the
8718	* hypervisor debugger and should return
8719	* VINF_EM_DBG_STEPPED if the event was dispatched
8720	* directly.
8721	*/
8722	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
8723	{
8724	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8725	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8726
8727	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8728	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8729
8730	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8731	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8732	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8733	Assert(!TRPMHasTrap(pVCpu));
8734
8735	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8736	if (pVCpu->hm.s.Event.fPending)
8737	{
8738	/*
8739	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8740	* pending even while injecting an event and in this case, we want a VM-exit as soon as
8741	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
8742	*
8743	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8744	*/
8745	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8746	#ifdef VBOX_STRICT
8747	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8748	{
8749	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
8750	Assert(!fBlockInt);
8751	Assert(!fBlockSti);
8752	Assert(!fBlockMovSS);
8753	}
8754	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
8755	{
8756	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8757	Assert(!fBlockSti);
8758	Assert(!fBlockMovSS);
8759	Assert(!fBlockNmi);
8760	}
8761	#endif
8762	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8763	uIntType));
8764
8765	/*
8766	* Inject the event and get any changes to the guest-interruptibility state.
8767	*
8768	* The guest-interruptibility state may need to be updated if we inject the event
8769	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
8770	*/
8771	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
8772	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8773
8774	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8775	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8776	else
8777	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8778	}
8779
8780	/*
8781	* Update the guest-interruptibility state.
8782	*
8783	* This is required for the real-on-v86 software interrupt injection case above, as well as
8784	* updates to the guest state from ring-3 or IEM/REM.
8785	*/
8786	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
8787	AssertRCReturn(rc, rc);
8788
8789	/*
8790	* There's no need to clear the VM-entry interruption-information field here if we're not
8791	* injecting anything. VT-x clears the valid bit on every VM-exit.
8792	*
8793	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8794	*/
8795
8796	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8797	NOREF(fBlockMovSS); NOREF(fBlockSti);
8798	return rcStrict;
8799	}
8800
8801
8802	/**
8803	* Enters the VT-x session.
8804	*
8805	* @returns VBox status code.
8806	* @param pVCpu The cross context virtual CPU structure.
8807	*/
8808	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
8809	{
8810	AssertPtr(pVCpu);
8811	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8812	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8813
8814	LogFlowFunc(("pVCpu=%p\n", pVCpu));
8815	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8816	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8817
8818	#ifdef VBOX_STRICT
8819	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8820	RTCCUINTREG uHostCR4 = ASMGetCR4();
8821	if (!(uHostCR4 & X86_CR4_VMXE))
8822	{
8823	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8824	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8825	}
8826	#endif
8827
8828	/*
8829	* Load the appropriate VMCS as the current and active one.
8830	*/
8831	PVMXVMCSINFO pVmcsInfo;
8832	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
8833	if (!fInNestedGuestMode)
8834	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
8835	else
8836	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
8837	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
8838	if (RT_SUCCESS(rc))
8839	{
8840	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
8841	pVCpu->hm.s.fLeaveDone = false;
8842	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8843
8844	/*
8845	* Do the EMT scheduled L1D flush here if needed.
8846	*/
8847	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8848	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8849	}
8850	return rc;
8851	}
8852
8853
8854	/**
8855	* The thread-context callback (only on platforms which support it).
8856	*
8857	* @param enmEvent The thread-context event.
8858	* @param pVCpu The cross context virtual CPU structure.
8859	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8860	* @thread EMT(pVCpu)
8861	*/
8862	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8863	{
8864	NOREF(fGlobalInit);
8865
8866	switch (enmEvent)
8867	{
8868	case RTTHREADCTXEVENT_OUT:
8869	{
8870	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8871	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8872	VMCPU_ASSERT_EMT(pVCpu);
8873
8874	/* No longjmps (logger flushes, locks) in this fragile context. */
8875	VMMRZCallRing3Disable(pVCpu);
8876	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8877
8878	/* Restore host-state (FPU, debug etc.) */
8879	if (!pVCpu->hm.s.fLeaveDone)
8880	{
8881	/*
8882	* Do -not- import the guest-state here as we might already be in the middle of importing
8883	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8884	*/
8885	hmR0VmxLeave(pVCpu, false /* fImportState */);
8886	pVCpu->hm.s.fLeaveDone = true;
8887	}
8888
8889	/* Leave HM context, takes care of local init (term). */
8890	int rc = HMR0LeaveCpu(pVCpu);
8891	AssertRC(rc);
8892
8893	/* Restore longjmp state. */
8894	VMMRZCallRing3Enable(pVCpu);
8895	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8896	break;
8897	}
8898
8899	case RTTHREADCTXEVENT_IN:
8900	{
8901	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8902	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8903	VMCPU_ASSERT_EMT(pVCpu);
8904
8905	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8906	VMMRZCallRing3Disable(pVCpu);
8907	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8908
8909	/* Initialize the bare minimum state required for HM. This takes care of
8910	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8911	int rc = hmR0EnterCpu(pVCpu);
8912	AssertRC(rc);
8913	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8914	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8915
8916	/* Load the active VMCS as the current one. */
8917	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8918	rc = hmR0VmxLoadVmcs(pVmcsInfo);
8919	AssertRC(rc);
8920	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8921	pVCpu->hm.s.fLeaveDone = false;
8922
8923	/* Do the EMT scheduled L1D flush if needed. */
8924	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8925	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8926
8927	/* Restore longjmp state. */
8928	VMMRZCallRing3Enable(pVCpu);
8929	break;
8930	}
8931
8932	default:
8933	break;
8934	}
8935	}
8936
8937
8938	/**
8939	* Exports the host state into the VMCS host-state area.
8940	* Sets up the VM-exit MSR-load area.
8941	*
8942	* The CPU state will be loaded from these fields on every successful VM-exit.
8943	*
8944	* @returns VBox status code.
8945	* @param pVCpu The cross context virtual CPU structure.
8946	*
8947	* @remarks No-long-jump zone!!!
8948	*/
8949	static int hmR0VmxExportHostState(PVMCPU pVCpu)
8950	{
8951	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8952
8953	int rc = VINF_SUCCESS;
8954	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8955	{
8956	rc = hmR0VmxExportHostControlRegs();
8957	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8958
8959	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8960	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8961
8962	rc = hmR0VmxExportHostMsrs(pVCpu);
8963	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8964
8965	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8966	}
8967	return rc;
8968	}
8969
8970
8971	/**
8972	* Saves the host state in the VMCS host-state.
8973	*
8974	* @returns VBox status code.
8975	* @param pVCpu The cross context virtual CPU structure.
8976	*
8977	* @remarks No-long-jump zone!!!
8978	*/
8979	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
8980	{
8981	AssertPtr(pVCpu);
8982	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8983
8984	/*
8985	* Export the host state here while entering HM context.
8986	* When thread-context hooks are used, we might get preempted and have to re-save the host
8987	* state but most of the time we won't be, so do it here before we disable interrupts.
8988	*/
8989	return hmR0VmxExportHostState(pVCpu);
8990	}
8991
8992
8993	/**
8994	* Exports the guest state into the VMCS guest-state area.
8995	*
8996	* The will typically be done before VM-entry when the guest-CPU state and the
8997	* VMCS state may potentially be out of sync.
8998	*
8999	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9000	* VM-entry controls.
9001	* Sets up the appropriate VMX non-root function to execute guest code based on
9002	* the guest CPU mode.
9003	*
9004	* @returns VBox strict status code.
9005	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9006	* without unrestricted guest execution and the VMMDev is not presently
9007	* mapped (e.g. EFI32).
9008	*
9009	* @param pVCpu The cross context virtual CPU structure.
9010	* @param pVmxTransient The VMX-transient structure.
9011	*
9012	* @remarks No-long-jump zone!!!
9013	*/
9014	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9015	{
9016	AssertPtr(pVCpu);
9017	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9018	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9019
9020	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9021
9022	/*
9023	* Determine real-on-v86 mode.
9024	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9025	*/
9026	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9027	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9028	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9029	pVmcsInfo->RealMode. fRealOnV86Active = false;
9030	else
9031	{
9032	Assert(!pVmxTransient->fIsNestedGuest);
9033	pVmcsInfo->RealMode.fRealOnV86Active = true;
9034	}
9035
9036	/*
9037	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9038	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9039	*/
9040	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9041	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9042	* be a need to evaluate this everytime since I'm pretty sure we intercept
9043	* all guest paging mode changes. */
9044	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9045	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9046
9047	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9048	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9049
9050	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9051	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9052
9053	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9054	if (rcStrict == VINF_SUCCESS)
9055	{ /* likely */ }
9056	else
9057	{
9058	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9059	return rcStrict;
9060	}
9061
9062	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9063	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9064
9065	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9066	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9067
9068	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9069	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9070
9071	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9072	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9073
9074	rc = hmR0VmxExportGuestRip(pVCpu);
9075	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9076	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9077	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9078
9079	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9080	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9081	\| HM_CHANGED_GUEST_CR2
9082	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9083	\| HM_CHANGED_GUEST_X87
9084	\| HM_CHANGED_GUEST_SSE_AVX
9085	\| HM_CHANGED_GUEST_OTHER_XSAVE
9086	\| HM_CHANGED_GUEST_XCRx
9087	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9088	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9089	\| HM_CHANGED_GUEST_TSC_AUX
9090	\| HM_CHANGED_GUEST_OTHER_MSRS
9091	\| HM_CHANGED_GUEST_HWVIRT /* More accurate PLE handling someday? */
9092	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9093
9094	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9095	return rc;
9096	}
9097
9098
9099	/**
9100	* Exports the state shared between the host and guest into the VMCS.
9101	*
9102	* @param pVCpu The cross context virtual CPU structure.
9103	* @param pVmxTransient The VMX-transient structure.
9104	*
9105	* @remarks No-long-jump zone!!!
9106	*/
9107	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9108	{
9109	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9110	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9111
9112	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9113	{
9114	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9115	AssertRC(rc);
9116	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9117
9118	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9119	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9120	{
9121	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9122	AssertRC(rc);
9123	}
9124	}
9125
9126	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9127	{
9128	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9129	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9130	}
9131
9132	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9133	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9134	}
9135
9136
9137	/**
9138	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9139	*
9140	* @returns Strict VBox status code (i.e. informational status codes too).
9141	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9142	* without unrestricted guest execution and the VMMDev is not presently
9143	* mapped (e.g. EFI32).
9144	*
9145	* @param pVCpu The cross context virtual CPU structure.
9146	* @param pVmxTransient The VMX-transient structure.
9147	*
9148	* @remarks No-long-jump zone!!!
9149	*/
9150	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9151	{
9152	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9153	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9154	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9155
9156	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9157	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9158	#endif
9159
9160	/*
9161	* For many exits it's only RIP that changes and hence try to export it first
9162	* without going through a lot of change flag checks.
9163	*/
9164	VBOXSTRICTRC rcStrict;
9165	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9166	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9167	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9168	{
9169	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9170	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9171	{ /* likely */}
9172	else
9173	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9174	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9175	}
9176	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9177	{
9178	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9179	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9180	{ /* likely */}
9181	else
9182	{
9183	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9184	VBOXSTRICTRC_VAL(rcStrict)));
9185	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9186	return rcStrict;
9187	}
9188	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9189	}
9190	else
9191	rcStrict = VINF_SUCCESS;
9192
9193	#ifdef VBOX_STRICT
9194	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9195	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9196	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9197	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9198	("fCtxChanged=%#RX64\n", fCtxChanged));
9199	#endif
9200	return rcStrict;
9201	}
9202
9203
9204	/**
9205	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9206	* and update error record fields accordingly.
9207	*
9208	* @return VMX_IGS_* return codes.
9209	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9210	* wrong with the guest state.
9211	*
9212	* @param pVCpu The cross context virtual CPU structure.
9213	* @param pVmcsInfo The VMCS info. object.
9214	*
9215	* @remarks This function assumes our cache of the VMCS controls
9216	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9217	*/
9218	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
9219	{
9220	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9221	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9222	uError = (err); \
9223	break; \
9224	} else do { } while (0)
9225
9226	int rc;
9227	PVM pVM = pVCpu->CTX_SUFF(pVM);
9228	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9229	uint32_t uError = VMX_IGS_ERROR;
9230	uint32_t u32Val;
9231	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9232
9233	do
9234	{
9235	/*
9236	* CR0.
9237	*/
9238	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9239	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9240	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
9241	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9242	if (fUnrestrictedGuest)
9243	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
9244
9245	uint32_t u32GuestCr0;
9246	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
9247	AssertRCBreak(rc);
9248	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9249	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9250	if ( !fUnrestrictedGuest
9251	&& (u32GuestCr0 & X86_CR0_PG)
9252	&& !(u32GuestCr0 & X86_CR0_PE))
9253	{
9254	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9255	}
9256
9257	/*
9258	* CR4.
9259	*/
9260	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9261	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9262
9263	uint32_t u32GuestCr4;
9264	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
9265	AssertRCBreak(rc);
9266	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9267	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9268
9269	/*
9270	* IA32_DEBUGCTL MSR.
9271	*/
9272	uint64_t u64Val;
9273	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9274	AssertRCBreak(rc);
9275	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9276	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9277	{
9278	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9279	}
9280	uint64_t u64DebugCtlMsr = u64Val;
9281
9282	#ifdef VBOX_STRICT
9283	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9284	AssertRCBreak(rc);
9285	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9286	#endif
9287	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9288
9289	/*
9290	* RIP and RFLAGS.
9291	*/
9292	uint32_t u32Eflags;
9293	#if HC_ARCH_BITS == 64
9294	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9295	AssertRCBreak(rc);
9296	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9297	if ( !fLongModeGuest
9298	\|\| !pCtx->cs.Attr.n.u1Long)
9299	{
9300	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9301	}
9302	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9303	* must be identical if the "IA-32e mode guest" VM-entry
9304	* control is 1 and CS.L is 1. No check applies if the
9305	* CPU supports 64 linear-address bits. */
9306
9307	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9308	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9309	AssertRCBreak(rc);
9310	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9311	VMX_IGS_RFLAGS_RESERVED);
9312	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9313	u32Eflags = u64Val;
9314	#else
9315	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9316	AssertRCBreak(rc);
9317	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9318	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9319	#endif
9320
9321	if ( fLongModeGuest
9322	\|\| ( fUnrestrictedGuest
9323	&& !(u32GuestCr0 & X86_CR0_PE)))
9324	{
9325	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9326	}
9327
9328	uint32_t u32EntryInfo;
9329	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9330	AssertRCBreak(rc);
9331	if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9332	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9333	{
9334	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9335	}
9336
9337	/*
9338	* 64-bit checks.
9339	*/
9340	#if HC_ARCH_BITS == 64
9341	if (fLongModeGuest)
9342	{
9343	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9344	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9345	}
9346
9347	if ( !fLongModeGuest
9348	&& (u32GuestCr4 & X86_CR4_PCIDE))
9349	{
9350	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9351	}
9352
9353	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9354	* 51:32 beyond the processor's physical-address width are 0. */
9355
9356	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9357	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9358	{
9359	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9360	}
9361
9362	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9363	AssertRCBreak(rc);
9364	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9365
9366	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9367	AssertRCBreak(rc);
9368	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9369	#endif
9370
9371	/*
9372	* PERF_GLOBAL MSR.
9373	*/
9374	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9375	{
9376	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9377	AssertRCBreak(rc);
9378	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9379	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9380	}
9381
9382	/*
9383	* PAT MSR.
9384	*/
9385	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9386	{
9387	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9388	AssertRCBreak(rc);
9389	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9390	for (unsigned i = 0; i < 8; i++)
9391	{
9392	uint8_t u8Val = (u64Val & 0xff);
9393	if ( u8Val != 0 /* UC */
9394	&& u8Val != 1 /* WC */
9395	&& u8Val != 4 /* WT */
9396	&& u8Val != 5 /* WP */
9397	&& u8Val != 6 /* WB */
9398	&& u8Val != 7 /* UC- */)
9399	{
9400	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9401	}
9402	u64Val >>= 8;
9403	}
9404	}
9405
9406	/*
9407	* EFER MSR.
9408	*/
9409	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9410	{
9411	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9412	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9413	AssertRCBreak(rc);
9414	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9415	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9416	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9417	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9418	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9419	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9420	* iemVmxVmentryCheckGuestState(). */
9421	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9422	\|\| !(u32GuestCr0 & X86_CR0_PG)
9423	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9424	VMX_IGS_EFER_LMA_LME_MISMATCH);
9425	}
9426
9427	/*
9428	* Segment registers.
9429	*/
9430	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9431	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9432	if (!(u32Eflags & X86_EFL_VM))
9433	{
9434	/* CS */
9435	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9436	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9437	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9438	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9439	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9440	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9441	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9442	/* CS cannot be loaded with NULL in protected mode. */
9443	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9444	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9445	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9446	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9447	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9448	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9449	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9450	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9451	else
9452	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9453
9454	/* SS */
9455	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9456	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9457	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9458	if ( !(pCtx->cr0 & X86_CR0_PE)
9459	\|\| pCtx->cs.Attr.n.u4Type == 3)
9460	{
9461	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9462	}
9463	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9464	{
9465	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9466	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9467	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9468	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9469	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9470	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9471	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9472	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9473	}
9474
9475	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9476	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9477	{
9478	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9479	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9480	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9481	\|\| pCtx->ds.Attr.n.u4Type > 11
9482	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9483	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9484	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9485	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9486	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9487	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9488	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9489	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9490	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9491	}
9492	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9493	{
9494	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9495	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9496	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9497	\|\| pCtx->es.Attr.n.u4Type > 11
9498	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9499	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9500	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9501	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9502	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9503	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9504	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9505	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9506	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9507	}
9508	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9509	{
9510	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9511	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9512	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9513	\|\| pCtx->fs.Attr.n.u4Type > 11
9514	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9515	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9516	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9517	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9518	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9519	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9520	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9521	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9522	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9523	}
9524	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9525	{
9526	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9527	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9528	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9529	\|\| pCtx->gs.Attr.n.u4Type > 11
9530	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9531	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9532	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9533	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9534	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9535	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9536	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9537	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9538	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9539	}
9540	/* 64-bit capable CPUs. */
9541	#if HC_ARCH_BITS == 64
9542	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9543	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9544	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9545	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9546	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9547	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9548	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9549	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9550	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9551	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9552	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9553	#endif
9554	}
9555	else
9556	{
9557	/* V86 mode checks. */
9558	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9559	if (pVmcsInfo->RealMode.fRealOnV86Active)
9560	{
9561	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9562	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9563	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9564	}
9565	else
9566	{
9567	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9568	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9569	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9570	}
9571
9572	/* CS */
9573	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9574	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9575	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9576	/* SS */
9577	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9578	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9579	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9580	/* DS */
9581	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9582	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9583	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9584	/* ES */
9585	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9586	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9587	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9588	/* FS */
9589	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9590	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9591	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9592	/* GS */
9593	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9594	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9595	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9596	/* 64-bit capable CPUs. */
9597	#if HC_ARCH_BITS == 64
9598	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9599	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9600	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9601	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9602	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9603	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9604	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9605	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9606	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9607	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9608	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9609	#endif
9610	}
9611
9612	/*
9613	* TR.
9614	*/
9615	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9616	/* 64-bit capable CPUs. */
9617	#if HC_ARCH_BITS == 64
9618	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9619	#endif
9620	if (fLongModeGuest)
9621	{
9622	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9623	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9624	}
9625	else
9626	{
9627	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9628	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9629	VMX_IGS_TR_ATTR_TYPE_INVALID);
9630	}
9631	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9632	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9633	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9634	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9635	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9636	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9637	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9638	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9639
9640	/*
9641	* GDTR and IDTR.
9642	*/
9643	#if HC_ARCH_BITS == 64
9644	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9645	AssertRCBreak(rc);
9646	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9647
9648	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9649	AssertRCBreak(rc);
9650	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9651	#endif
9652
9653	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9654	AssertRCBreak(rc);
9655	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9656
9657	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9658	AssertRCBreak(rc);
9659	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9660
9661	/*
9662	* Guest Non-Register State.
9663	*/
9664	/* Activity State. */
9665	uint32_t u32ActivityState;
9666	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9667	AssertRCBreak(rc);
9668	HMVMX_CHECK_BREAK( !u32ActivityState
9669	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9670	VMX_IGS_ACTIVITY_STATE_INVALID);
9671	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9672	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9673	uint32_t u32IntrState;
9674	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9675	AssertRCBreak(rc);
9676	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9677	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9678	{
9679	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9680	}
9681
9682	/** @todo Activity state and injecting interrupts. Left as a todo since we
9683	* currently don't use activity states but ACTIVE. */
9684
9685	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9686	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9687
9688	/* Guest interruptibility-state. */
9689	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9690	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9691	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9692	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9693	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9694	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9695	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9696	if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
9697	{
9698	if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9699	{
9700	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9701	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9702	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9703	}
9704	else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9705	{
9706	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9707	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9708	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9709	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9710	}
9711	}
9712	/** @todo Assumes the processor is not in SMM. */
9713	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9714	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9715	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9716	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9717	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9718	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9719	&& VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9720	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9721	{
9722	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
9723	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9724	}
9725
9726	/* Pending debug exceptions. */
9727	#if HC_ARCH_BITS == 64
9728	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
9729	AssertRCBreak(rc);
9730	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9731	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9732	u32Val = u64Val; /* For pending debug exceptions checks below. */
9733	#else
9734	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
9735	AssertRCBreak(rc);
9736	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9737	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9738	#endif
9739
9740	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9741	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
9742	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9743	{
9744	if ( (u32Eflags & X86_EFL_TF)
9745	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9746	{
9747	/* Bit 14 is PendingDebug.BS. */
9748	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9749	}
9750	if ( !(u32Eflags & X86_EFL_TF)
9751	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9752	{
9753	/* Bit 14 is PendingDebug.BS. */
9754	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9755	}
9756	}
9757
9758	/* VMCS link pointer. */
9759	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9760	AssertRCBreak(rc);
9761	if (u64Val != UINT64_C(0xffffffffffffffff))
9762	{
9763	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9764	/** @todo Bits beyond the processor's physical-address width MBZ. */
9765	/** @todo 32-bit located in memory referenced by value of this field (as a
9766	* physical address) must contain the processor's VMCS revision ID. */
9767	/** @todo SMM checks. */
9768	}
9769
9770	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9771	* not using nested paging? */
9772	if ( pVM->hm.s.fNestedPaging
9773	&& !fLongModeGuest
9774	&& CPUMIsGuestInPAEModeEx(pCtx))
9775	{
9776	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9777	AssertRCBreak(rc);
9778	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9779
9780	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9781	AssertRCBreak(rc);
9782	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9783
9784	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9785	AssertRCBreak(rc);
9786	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9787
9788	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9789	AssertRCBreak(rc);
9790	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9791	}
9792
9793	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9794	if (uError == VMX_IGS_ERROR)
9795	uError = VMX_IGS_REASON_NOT_FOUND;
9796	} while (0);
9797
9798	pVCpu->hm.s.u32HMError = uError;
9799	return uError;
9800
9801	#undef HMVMX_ERROR_BREAK
9802	#undef HMVMX_CHECK_BREAK
9803	}
9804
9805
9806	/**
9807	* Setup the APIC-access page for virtualizing APIC access.
9808	*
9809	* This can cause a longjumps to R3 due to the acquisition of the PGM lock, hence
9810	* this not done as part of exporting guest state, see @bugref{8721}.
9811	*
9812	* @returns VBox status code.
9813	* @param pVCpu The cross context virtual CPU structure.
9814	*/
9815	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
9816	{
9817	PVM pVM = pVCpu->CTX_SUFF(pVM);
9818	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
9819
9820	Assert(PDMHasApic(pVM));
9821	Assert(u64MsrApicBase);
9822
9823	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
9824	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
9825
9826	/* Unalias any existing mapping. */
9827	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
9828	AssertRCReturn(rc, rc);
9829
9830	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
9831	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
9832	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
9833	AssertRCReturn(rc, rc);
9834
9835	/* Update the per-VCPU cache of the APIC base MSR. */
9836	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
9837	return VINF_SUCCESS;
9838	}
9839
9840
9841	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9842	/**
9843	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
9844	* nested-guest using hardware-assisted VMX.
9845	*
9846	* @param pVCpu The cross context virtual CPU structure.
9847	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
9848	* @param pVmcsInfoGst The guest VMCS info. object.
9849	*/
9850	static void hmR0VmxMergeMsrBitmapNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
9851	{
9852	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
9853	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
9854	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
9855	Assert(pu64MsrBitmapNstGst);
9856	Assert(pu64MsrBitmapGst);
9857	Assert(pu64MsrBitmap);
9858
9859	/*
9860	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
9861	* MSR that is intercepted by the guest is also intercepted while executing the
9862	* nested-guest using hardware-assisted VMX.
9863	*/
9864	uint32_t const cbFrag = sizeof(uint64_t);
9865	uint32_t const cFrags = X86_PAGE_4K_SIZE / cbFrag;
9866	for (uint32_t i = 0; i <= cFrags; i++)
9867	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
9868	}
9869
9870
9871	/**
9872	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
9873	* hardware-assisted VMX execution of the nested-guest.
9874	*
9875	* For a guest, we don't modify these controls once we set up the VMCS.
9876	*
9877	* For nested-guests since the guest hypervisor provides these controls on every
9878	* nested-guest VM-entry and could potentially change them everytime we need to
9879	* merge them before every nested-guest VM-entry.
9880	*
9881	* @returns VBox status code.
9882	* @param pVCpu The cross context virtual CPU structure.
9883	*/
9884	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
9885	{
9886	PVM pVM = pVCpu->CTX_SUFF(pVM);
9887	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
9888	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
9889	Assert(pVmcsNstGst);
9890
9891	/*
9892	* Merge the controls with the requirements of the guest VMCS.
9893	*
9894	* We do not need to validate the nested-guest VMX features specified in the
9895	* nested-guest VMCS with the features supported by the physical CPU as it's
9896	* already done by the VMLAUNCH/VMRESUME instruction emulation.
9897	*
9898	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the
9899	* guest are derived from the VMX features supported by the physical CPU.
9900	*/
9901
9902	/* Pin-based VM-execution controls. */
9903	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
9904
9905	/* Processor-based VM-execution controls. */
9906	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
9907	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
9908	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
9909	\| VMX_PROC_CTLS_USE_TPR_SHADOW
9910	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
9911
9912	/* Secondary processor-based VM-execution controls. */
9913	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
9914	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
9915	\| VMX_PROC_CTLS2_INVPCID
9916	\| VMX_PROC_CTLS2_RDTSCP
9917	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
9918	\| VMX_PROC_CTLS2_APIC_REG_VIRT
9919	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
9920	\| VMX_PROC_CTLS2_VMFUNC));
9921
9922	/*
9923	* VM-entry controls:
9924	* These controls contains state that depends on the nested-guest state (primarily
9925	* EFER MSR) and is thus not constant through VMLAUNCH/VMRESUME and the nested-guest
9926	* VM-exit. Although the nested-hypervisor cannot change it, we need to in order to
9927	* properly continue executing the nested-guest if the EFER MSR changes but does not
9928	* cause a nested-guest VM-exits.
9929	*
9930	* VM-exit controls:
9931	* These controls specify the host state on return. We cannot use the controls from
9932	* the nested-hypervisor state as is as it would contain the guest state rather than
9933	* the host state. Since the host state is subject to change (e.g. preemption, trips
9934	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
9935	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
9936	*
9937	* VM-entry MSR-load:
9938	* The guest MSRs from the VM-entry MSR-load area are already loaded into the
9939	* guest-CPU context by the VMLAUNCH/VMRESUME instruction emulation.
9940	*
9941	* VM-exit MSR-store:
9942	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU
9943	* context back into the VM-exit MSR-store area.
9944	*
9945	* VM-exit MSR-load areas:
9946	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence,
9947	* we can entirely ignore what the nested-hypervisor wants to load here.
9948	*/
9949
9950	/*
9951	* Exception bitmap.
9952	*
9953	* We could remove #UD from the guest bitmap and merge it with the nested-guest
9954	* bitmap here (and avoid doing anything while exporting nested-guest state), but to
9955	* keep the code more flexible if intercepting exceptions become more dynamic in
9956	* the future we do it as part of exporting the nested-guest state.
9957	*/
9958	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
9959
9960	/*
9961	* CR0/CR4 guest/host mask.
9962	*
9963	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest
9964	* must cause VM-exits, so we need to merge them here.
9965	*/
9966	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
9967	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
9968
9969	/*
9970	* Page-fault error-code mask and match.
9971	*
9972	* Although we require unrestricted guest execution (and thereby nested-paging) for
9973	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
9974	* normally intercept #PFs, it might intercept them for debugging purposes.
9975	*
9976	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF
9977	* filters. If the outer guest is intercepting #PFs we must intercept all #PFs.
9978	*/
9979	uint32_t u32XcptPFMask;
9980	uint32_t u32XcptPFMatch;
9981	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
9982	{
9983	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
9984	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
9985	}
9986	else
9987	{
9988	u32XcptPFMask = 0;
9989	u32XcptPFMatch = 0;
9990	}
9991
9992	/*
9993	* Pause-Loop exiting.
9994	*/
9995	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
9996	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
9997
9998	/*
9999	* I/O Bitmap.
10000	*
10001	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we
10002	* always intercept all I/O port accesses.
10003	*/
10004	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10005
10006	/*
10007	* APIC-access page.
10008	*
10009	* The APIC-access page address has already been initialized while setting up the
10010	* nested-guest VMCS. In theory, even if the guest-physical address is invalid, it
10011	* should not be on any consequence to the host or to the guest for that matter, but
10012	* we only accept valid addresses verified by the VMLAUNCH/VMRESUME instruction
10013	* emulation to keep it simple.
10014	*/
10015
10016	/*
10017	* Virtual-APIC page and TPR threshold.
10018	*
10019	* We shall use the host-physical address of the virtual-APIC page in guest memory directly.
10020	* For this reason, we can access the virtual-APIC page of the nested-guest only using
10021	* PGM physical handlers as we must not assume a kernel virtual-address mapping exists and
10022	* requesting PGM for a mapping could be expensive/resource intensive (PGM mapping cache).
10023	*/
10024	RTHCPHYS HCPhysVirtApic = NIL_RTHCPHYS;
10025	uint32_t const u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10026	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10027	{
10028	int rc = PGMPhysGCPhys2HCPhys(pVM, pVmcsNstGst->u64AddrVirtApic.u, &HCPhysVirtApic);
10029
10030	/*
10031	* If the guest hypervisor has loaded crap into the virtual-APIC page field
10032	* we would fail to obtain a valid host-physical address for its guest-physical
10033	* address.
10034	*
10035	* We currently do not support this scenario. Maybe in the future if there is a
10036	* pressing need we can explore making this particular set of conditions work.
10037	* Right now we just cause a VM-entry failure.
10038	*
10039	* This has already been checked by VMLAUNCH/VMRESUME instruction emulation,
10040	* so should not really failure at the moment.
10041	*/
10042	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10043	}
10044	else
10045	{
10046	/*
10047	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10048	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10049	* be taken care of by EPT/shadow paging.
10050	*/
10051	if (pVM->hm.s.fAllow64BitGuests)
10052	{
10053	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10054	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10055	}
10056	}
10057
10058	/*
10059	* Validate basic assumptions.
10060	*/
10061	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10062	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10063	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10064	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10065
10066	/*
10067	* Commit it to the nested-guest VMCS.
10068	*/
10069	int rc = VINF_SUCCESS;
10070	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10071	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10072	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10073	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10074	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10075	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10076	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10077	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10078	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10079	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10080	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10081	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10082	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10083	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10084	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10085	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10086	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10087	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10088	{
10089	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10090	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10091	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10092	}
10093	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10094	{
10095	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10096	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10097	}
10098	AssertRCReturn(rc, rc);
10099
10100	/*
10101	* Update the nested-guest VMCS cache.
10102	*/
10103	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10104	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10105	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10106	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10107	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10108	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10109	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10110	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10111	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10112
10113	/*
10114	* MSR bitmap.
10115	*
10116	* The MSR bitmap address has already been initialized while setting up the
10117	* nested-guest VMCS, here we need to merge the MSR bitmaps.
10118	*/
10119	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10120	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10121
10122	return VINF_SUCCESS;
10123	}
10124	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10125
10126
10127	/**
10128	* Does the preparations before executing guest code in VT-x.
10129	*
10130	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10131	* recompiler/IEM. We must be cautious what we do here regarding committing
10132	* guest-state information into the VMCS assuming we assuredly execute the
10133	* guest in VT-x mode.
10134	*
10135	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10136	* the common-state (TRPM/forceflags), we must undo those changes so that the
10137	* recompiler/IEM can (and should) use them when it resumes guest execution.
10138	* Otherwise such operations must be done when we can no longer exit to ring-3.
10139	*
10140	* @returns Strict VBox status code (i.e. informational status codes too).
10141	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10142	* have been disabled.
10143	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10144	* double-fault into the guest.
10145	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10146	* dispatched directly.
10147	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10148	*
10149	* @param pVCpu The cross context virtual CPU structure.
10150	* @param pVmxTransient The VMX-transient structure.
10151	* @param fStepping Whether we are single-stepping the guest in the
10152	* hypervisor debugger. Makes us ignore some of the reasons
10153	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10154	* if event dispatching took place.
10155	*/
10156	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10157	{
10158	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10159
10160	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10161	if (pVmxTransient->fIsNestedGuest)
10162	{
10163	RT_NOREF2(pVCpu, fStepping);
10164	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10165	return VINF_EM_RESCHEDULE_REM;
10166	}
10167	#endif
10168
10169	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10170	PGMRZDynMapFlushAutoSet(pVCpu);
10171	#endif
10172
10173	/*
10174	* Check and process force flag actions, some of which might require us to go back to ring-3.
10175	*/
10176	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10177	if (rcStrict == VINF_SUCCESS)
10178	{ /* FFs don't get set all the time. */ }
10179	else
10180	return rcStrict;
10181
10182	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10183	/*
10184	* Switch to the nested-guest VMCS as we may have transitioned into executing
10185	* the nested-guest without leaving ring-0. Otherwise, if we came from ring-3
10186	* we would load the nested-guest VMCS while entering the VMX ring-0 session.
10187	*
10188	* We do this as late as possible to minimize (though not completely remove)
10189	* clearing/loading VMCS again due to premature trips to ring-3 above.
10190	*/
10191	if (pVmxTransient->fIsNestedGuest)
10192	{
10193	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10194	{
10195	/*
10196	* Ensure we have synced everything from the guest VMCS and also flag that
10197	* that we need to export the full (nested) guest-CPU context to the
10198	* nested-guest VMCS.
10199	*/
10200	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
10201	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST);
10202
10203	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
10204	int rc = hmR0VmxSwitchVmcs(&pVCpu->hm.s.vmx.VmcsInfo, &pVCpu->hm.s.vmx.VmcsInfoNstGst);
10205	if (RT_LIKELY(rc == VINF_SUCCESS))
10206	{
10207	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = true;
10208	ASMSetFlags(fEFlags);
10209	pVmxTransient->pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10210
10211	/*
10212	* We use a different VM-exit MSR-store area for the nested-guest. Hence,
10213	* flag that we need to update the host MSR values there. Even if we decide
10214	* in the future to share the VM-exit MSR-store area page with the guest,
10215	* if its content differs, we would have to update the host MSRs anyway.
10216	*/
10217	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
10218	Assert(!pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer); /** @todo NSTVMX: Paranoia remove later. */
10219	}
10220	else
10221	{
10222	ASMSetFlags(fEFlags);
10223	return rc;
10224	}
10225	}
10226
10227	/*
10228	* Merge guest VMCS controls with the nested-guest VMCS controls.
10229	*
10230	* Even if we have not executed the guest prior to this (e.g. when resuming
10231	* from a saved state), we should be okay with merging controls as we
10232	* initialize the guest VMCS controls as part of VM setup phase.
10233	*/
10234	if (!pVCpu->hm.s.vmx.fMergedNstGstCtls)
10235	{
10236	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10237	AssertRCReturn(rc, rc);
10238	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10239	}
10240	}
10241	#endif
10242
10243	/*
10244	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10245	* We look at the guest VMCS control here as we always set it when supported by
10246	* the physical CPU. Looking at the nested-guest control here would not be
10247	* possible because they are not merged yet.
10248	*/
10249	PVM pVM = pVCpu->CTX_SUFF(pVM);
10250	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10251	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10252	&& (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10253	&& PDMHasApic(pVM))
10254	{
10255	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10256	AssertRCReturn(rc, rc);
10257	}
10258
10259	/*
10260	* Evaluate events to be injected into the guest.
10261	*
10262	* Events in TRPM can be injected without inspecting the guest state.
10263	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10264	* guest to cause a VM-exit the next time they are ready to receive the event.
10265	*/
10266	if (TRPMHasTrap(pVCpu))
10267	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10268
10269	uint32_t fIntrState;
10270	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10271
10272	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10273	/*
10274	* While evaluating pending events if something failed (unlikely) or if we were
10275	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10276	*/
10277	if ( rcStrict != VINF_SUCCESS
10278	\|\| ( pVmxTransient->fIsNestedGuest
10279	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)))
10280	return rcStrict;
10281	#endif
10282
10283	/*
10284	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10285	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10286	* also result in triple-faulting the VM.
10287	*
10288	* The above does not apply when executing a nested-guest (since unrestricted guest execution
10289	* is a requirement) regardless doing it avoid duplicating code elsewhere.
10290	*/
10291	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10292	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10293	{ /* likely */ }
10294	else
10295	{
10296	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10297	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10298	return rcStrict;
10299	}
10300
10301	/*
10302	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10303	* import CR3 themselves. We will need to update them here, as even as late as the above
10304	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10305	* the below force flags to be set.
10306	*/
10307	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10308	{
10309	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10310	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10311	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10312	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10313	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10314	}
10315	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10316	{
10317	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10318	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10319	}
10320
10321	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10322	/* Paranoia. */
10323	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10324	#endif
10325
10326	/*
10327	* No longjmps to ring-3 from this point on!!!
10328	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10329	* This also disables flushing of the R0-logger instance (if any).
10330	*/
10331	VMMRZCallRing3Disable(pVCpu);
10332
10333	/*
10334	* Export the guest state bits.
10335	*
10336	* We cannot perform longjmps while loading the guest state because we do not preserve the
10337	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10338	* CPU migration.
10339	*
10340	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10341	* registers. Hence, loading of the guest state needs to be done -after- injection of events.
10342	*/
10343	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10344	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10345	{ /* likely */ }
10346	else
10347	{
10348	VMMRZCallRing3Enable(pVCpu);
10349	return rcStrict;
10350	}
10351
10352	/*
10353	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10354	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10355	* preemption disabled for a while. Since this is purely to aid the
10356	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10357	* disable interrupt on NT.
10358	*
10359	* We need to check for force-flags that could've possible been altered since we last
10360	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10361	* see @bugref{6398}).
10362	*
10363	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10364	* to ring-3 before executing guest code.
10365	*/
10366	pVmxTransient->fEFlags = ASMIntDisableFlags();
10367
10368	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10369	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10370	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10371	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10372	{
10373	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10374	{
10375	pVCpu->hm.s.Event.fPending = false;
10376
10377	/*
10378	* We've injected any pending events. This is really the point of no return (to ring-3).
10379	*
10380	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10381	* returns from this function, so don't enable them here.
10382	*/
10383	return VINF_SUCCESS;
10384	}
10385
10386	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10387	rcStrict = VINF_EM_RAW_INTERRUPT;
10388	}
10389	else
10390	{
10391	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10392	rcStrict = VINF_EM_RAW_TO_R3;
10393	}
10394
10395	ASMSetFlags(pVmxTransient->fEFlags);
10396	VMMRZCallRing3Enable(pVCpu);
10397
10398	return rcStrict;
10399	}
10400
10401
10402	/**
10403	* Final preparations before executing guest code using hardware-assisted VMX.
10404	*
10405	* We can no longer get preempted to a different host CPU and there are no returns
10406	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10407	* failures), this function is not intended to fail sans unrecoverable hardware
10408	* errors.
10409	*
10410	* @param pVCpu The cross context virtual CPU structure.
10411	* @param pVmxTransient The VMX-transient structure.
10412	*
10413	* @remarks Called with preemption disabled.
10414	* @remarks No-long-jump zone!!!
10415	*/
10416	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10417	{
10418	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10419	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10420	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10421	Assert(!pVCpu->hm.s.Event.fPending);
10422
10423	/*
10424	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10425	*/
10426	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10427	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10428
10429	PVM pVM = pVCpu->CTX_SUFF(pVM);
10430	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10431
10432	if (!CPUMIsGuestFPUStateActive(pVCpu))
10433	{
10434	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10435	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10436	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10437	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10438	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10439	}
10440
10441	/*
10442	* Re-save the host state bits as we may've been preempted (only happens when
10443	* thread-context hooks are used or when the VM start function changes).
10444	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10445	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10446	* see @bugref{8432}.
10447	*
10448	* This may also happen when switching to/from a nested-guest VMCS without leaving
10449	* ring-0.
10450	*/
10451	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10452	{
10453	int rc = hmR0VmxExportHostState(pVCpu);
10454	AssertRC(rc);
10455	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
10456	}
10457	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10458
10459	/*
10460	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10461	*/
10462	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10463	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10464	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10465
10466	/*
10467	* Store status of the shared guest/host debug state at the time of VM-entry.
10468	*/
10469	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
10470	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
10471	{
10472	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
10473	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
10474	}
10475	else
10476	#endif
10477	{
10478	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10479	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10480	}
10481
10482	/*
10483	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10484	* more than one conditional check. The post-run side of our code shall determine
10485	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10486	*/
10487	if (pVmcsInfo->pbVirtApic)
10488	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10489
10490	/*
10491	* Update the host MSRs values in the VM-exit MSR-load area.
10492	*/
10493	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10494	{
10495	if (pVmcsInfo->cExitMsrLoad > 0)
10496	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10497	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10498	}
10499
10500	/*
10501	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10502	* VMX-preemption timer based on the next virtual sync clock deadline.
10503	*/
10504	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10505	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10506	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10507	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10508	{
10509	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10510	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10511	}
10512
10513	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10514	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10515	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10516	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
10517
10518	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10519
10520	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10521	as we're about to start executing the guest . */
10522
10523	/*
10524	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10525	*
10526	* This is done this late as updating the TSC offsetting/preemption timer above
10527	* figures out if we can skip intercepting RDTSCP by calculating the number of
10528	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10529	*/
10530	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10531	{
10532	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10533	{
10534	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10535	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10536	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10537	AssertRC(rc);
10538	}
10539	else
10540	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
10541	}
10542
10543	#ifdef VBOX_STRICT
10544	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo);
10545	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10546	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo));
10547	#endif
10548
10549	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10550	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10551	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs. */
10552	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10553	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10554	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10555	#endif
10556	}
10557
10558
10559	/**
10560	* First C routine invoked after running guest code using hardware-assisted VMX.
10561	*
10562	* @param pVCpu The cross context virtual CPU structure.
10563	* @param pVmxTransient The VMX-transient structure.
10564	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10565	*
10566	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10567	*
10568	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10569	* unconditionally when it is safe to do so.
10570	*/
10571	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10572	{
10573	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10574
10575	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10576	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10577	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10578	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10579	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10580	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10581
10582	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10583	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10584	{
10585	uint64_t uGstTsc;
10586	if (!pVmxTransient->fIsNestedGuest)
10587	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10588	else
10589	{
10590	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10591	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10592	}
10593	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10594	}
10595
10596	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10597	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
10598	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10599
10600	#if HC_ARCH_BITS == 64
10601	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10602	#endif
10603	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
10604	/* The 64-on-32 switcher maintains VMCS-launch state on its own
10605	and we need to leave it alone here. */
10606	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
10607	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10608	#else
10609	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10610	#endif
10611	#ifdef VBOX_STRICT
10612	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10613	#endif
10614	Assert(!ASMIntAreEnabled());
10615	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10616	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10617
10618	/*
10619	* Save the basic VM-exit reason and check if the VM-entry failed.
10620	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10621	*/
10622	uint32_t uExitReason;
10623	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
10624	AssertRC(rc);
10625	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
10626	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
10627
10628	/*
10629	* Check if VMLAUNCH/VMRESUME succeeded.
10630	* If this failed, we cause a guru meditation and cease further execution.
10631	*/
10632	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
10633	{
10634	/*
10635	* Update the VM-exit history array here even if the VM-entry failed due to:
10636	* - Invalid guest state.
10637	* - MSR loading.
10638	* - Machine-check event.
10639	*
10640	* In any of the above cases we will still have a "valid" VM-exit reason
10641	* despite @a fVMEntryFailed being false.
10642	*
10643	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
10644	*
10645	* Note! We don't have CS or RIP at this point. Will probably address that later
10646	* by amending the history entry added here.
10647	*/
10648	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
10649	UINT64_MAX, uHostTsc);
10650
10651	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
10652	{
10653	VMMRZCallRing3Enable(pVCpu);
10654
10655	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10656	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10657
10658	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
10659	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10660	AssertRC(rc);
10661	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
10662	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
10663	AssertRC(rc);
10664	#else
10665	/*
10666	* Import the guest-interruptibility state always as we need it while evaluating
10667	* injecting events on re-entry.
10668	*
10669	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
10670	* checking for real-mode while exporting the state because all bits that cause
10671	* mode changes wrt CR0 are intercepted.
10672	*/
10673	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
10674	AssertRC(rc);
10675	#endif
10676
10677	/*
10678	* Sync the TPR shadow with our APIC state.
10679	*/
10680	if ( !pVmxTransient->fIsNestedGuest
10681	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
10682	{
10683	Assert(pVmcsInfo->pbVirtApic);
10684	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
10685	{
10686	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
10687	AssertRC(rc);
10688	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
10689	}
10690	}
10691
10692	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10693	return;
10694	}
10695	}
10696	else
10697	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
10698
10699	VMMRZCallRing3Enable(pVCpu);
10700	}
10701
10702
10703	/**
10704	* Runs the guest code using hardware-assisted VMX the normal way.
10705	*
10706	* @returns VBox status code.
10707	* @param pVCpu The cross context virtual CPU structure.
10708	* @param pcLoops Pointer to the number of executed loops.
10709	*/
10710	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
10711	{
10712	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10713	Assert(pcLoops);
10714	Assert(*pcLoops <= cMaxResumeLoops);
10715
10716	VMXTRANSIENT VmxTransient;
10717	RT_ZERO(VmxTransient);
10718	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10719
10720	/* Paranoia. */
10721	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
10722	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10723
10724	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10725	for (;;)
10726	{
10727	Assert(!HMR0SuspendPending());
10728	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10729	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10730
10731	/*
10732	* Preparatory work for running nested-guest code, this may force us to
10733	* return to ring-3.
10734	*
10735	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10736	*/
10737	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10738	if (rcStrict != VINF_SUCCESS)
10739	break;
10740
10741	/* Interrupts are disabled at this point! */
10742	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10743	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10744	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10745	/* Interrupts are re-enabled at this point! */
10746
10747	/*
10748	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10749	*/
10750	if (RT_SUCCESS(rcRun))
10751	{ /* very likely */ }
10752	else
10753	{
10754	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10755	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10756	return rcRun;
10757	}
10758
10759	/*
10760	* Profile the VM-exit.
10761	*/
10762	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10763	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10764	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10765	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10766	HMVMX_START_EXIT_DISPATCH_PROF();
10767
10768	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10769
10770	/*
10771	* Handle the VM-exit.
10772	*/
10773	#ifdef HMVMX_USE_FUNCTION_TABLE
10774	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
10775	#else
10776	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
10777	#endif
10778	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10779	if (rcStrict == VINF_SUCCESS)
10780	{
10781	if (++(*pcLoops) <= cMaxResumeLoops)
10782	continue;
10783	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10784	rcStrict = VINF_EM_RAW_INTERRUPT;
10785	}
10786	break;
10787	}
10788
10789	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10790	return rcStrict;
10791	}
10792
10793	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10794	/**
10795	* Runs the nested-guest code using hardware-assisted VMX.
10796	*
10797	* @returns VBox status code.
10798	* @param pVCpu The cross context virtual CPU structure.
10799	* @param pcLoops Pointer to the number of executed loops.
10800	*
10801	* @sa hmR0VmxRunGuestCodeNormal().
10802	*/
10803	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
10804	{
10805	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10806	Assert(pcLoops);
10807	Assert(*pcLoops <= cMaxResumeLoops);
10808
10809	VMXTRANSIENT VmxTransient;
10810	RT_ZERO(VmxTransient);
10811	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10812	VmxTransient.fIsNestedGuest = true;
10813
10814	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10815	for (;;)
10816	{
10817	Assert(!HMR0SuspendPending());
10818	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10819	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10820
10821	/*
10822	* Preparatory work for running guest code, this may force us to
10823	* return to ring-3.
10824	*
10825	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10826	*/
10827	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10828	if (rcStrict != VINF_SUCCESS)
10829	break;
10830
10831	/* Interrupts are disabled at this point! */
10832	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10833	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10834	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10835	/* Interrupts are re-enabled at this point! */
10836
10837	/*
10838	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10839	*/
10840	if (RT_SUCCESS(rcRun))
10841	{ /* very likely */ }
10842	else
10843	{
10844	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10845	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10846	return rcRun;
10847	}
10848
10849	/*
10850	* Profile the VM-exit.
10851	*/
10852	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10853	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10854	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10855	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10856	HMVMX_START_EXIT_DISPATCH_PROF();
10857
10858	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10859
10860	/*
10861	* Handle the VM-exit.
10862	*/
10863	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
10864	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10865	if ( rcStrict == VINF_SUCCESS
10866	&& CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10867	{
10868	if (++(*pcLoops) <= cMaxResumeLoops)
10869	continue;
10870	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10871	rcStrict = VINF_EM_RAW_INTERRUPT;
10872	}
10873	break;
10874	}
10875
10876	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10877	return rcStrict;
10878	}
10879	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10880
10881
10882	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
10883	* probes.
10884	*
10885	* The following few functions and associated structure contains the bloat
10886	* necessary for providing detailed debug events and dtrace probes as well as
10887	* reliable host side single stepping. This works on the principle of
10888	* "subclassing" the normal execution loop and workers. We replace the loop
10889	* method completely and override selected helpers to add necessary adjustments
10890	* to their core operation.
10891	*
10892	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
10893	* any performance for debug and analysis features.
10894	*
10895	* @{
10896	*/
10897
10898	/**
10899	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
10900	* the debug run loop.
10901	*/
10902	typedef struct VMXRUNDBGSTATE
10903	{
10904	/** The RIP we started executing at. This is for detecting that we stepped. */
10905	uint64_t uRipStart;
10906	/** The CS we started executing with. */
10907	uint16_t uCsStart;
10908
10909	/** Whether we've actually modified the 1st execution control field. */
10910	bool fModifiedProcCtls : 1;
10911	/** Whether we've actually modified the 2nd execution control field. */
10912	bool fModifiedProcCtls2 : 1;
10913	/** Whether we've actually modified the exception bitmap. */
10914	bool fModifiedXcptBitmap : 1;
10915
10916	/** We desire the modified the CR0 mask to be cleared. */
10917	bool fClearCr0Mask : 1;
10918	/** We desire the modified the CR4 mask to be cleared. */
10919	bool fClearCr4Mask : 1;
10920	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
10921	uint32_t fCpe1Extra;
10922	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
10923	uint32_t fCpe1Unwanted;
10924	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
10925	uint32_t fCpe2Extra;
10926	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
10927	uint32_t bmXcptExtra;
10928	/** The sequence number of the Dtrace provider settings the state was
10929	* configured against. */
10930	uint32_t uDtraceSettingsSeqNo;
10931	/** VM-exits to check (one bit per VM-exit). */
10932	uint32_t bmExitsToCheck[3];
10933
10934	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
10935	uint32_t fProcCtlsInitial;
10936	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
10937	uint32_t fProcCtls2Initial;
10938	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
10939	uint32_t bmXcptInitial;
10940	} VMXRUNDBGSTATE;
10941	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
10942	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
10943
10944
10945	/**
10946	* Initializes the VMXRUNDBGSTATE structure.
10947	*
10948	* @param pVCpu The cross context virtual CPU structure of the
10949	* calling EMT.
10950	* @param pVmxTransient The VMX-transient structure.
10951	* @param pDbgState The debug state to initialize.
10952	*/
10953	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
10954	{
10955	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
10956	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
10957
10958	pDbgState->fModifiedProcCtls = false;
10959	pDbgState->fModifiedProcCtls2 = false;
10960	pDbgState->fModifiedXcptBitmap = false;
10961	pDbgState->fClearCr0Mask = false;
10962	pDbgState->fClearCr4Mask = false;
10963	pDbgState->fCpe1Extra = 0;
10964	pDbgState->fCpe1Unwanted = 0;
10965	pDbgState->fCpe2Extra = 0;
10966	pDbgState->bmXcptExtra = 0;
10967	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
10968	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
10969	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
10970	}
10971
10972
10973	/**
10974	* Updates the VMSC fields with changes requested by @a pDbgState.
10975	*
10976	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
10977	* immediately before executing guest code, i.e. when interrupts are disabled.
10978	* We don't check status codes here as we cannot easily assert or return in the
10979	* latter case.
10980	*
10981	* @param pVCpu The cross context virtual CPU structure.
10982	* @param pVmxTransient The VMX-transient structure.
10983	* @param pDbgState The debug state.
10984	*/
10985	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
10986	{
10987	/*
10988	* Ensure desired flags in VMCS control fields are set.
10989	* (Ignoring write failure here, as we're committed and it's just debug extras.)
10990	*
10991	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
10992	* there should be no stale data in pCtx at this point.
10993	*/
10994	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10995	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
10996	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
10997	{
10998	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
10999	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11000	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11001	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11002	pDbgState->fModifiedProcCtls = true;
11003	}
11004
11005	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11006	{
11007	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11008	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11009	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11010	pDbgState->fModifiedProcCtls2 = true;
11011	}
11012
11013	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11014	{
11015	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11016	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11017	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11018	pDbgState->fModifiedXcptBitmap = true;
11019	}
11020
11021	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11022	{
11023	pVmcsInfo->u64Cr0Mask = 0;
11024	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
11025	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11026	}
11027
11028	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11029	{
11030	pVmcsInfo->u64Cr4Mask = 0;
11031	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
11032	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11033	}
11034
11035	NOREF(pVCpu);
11036	}
11037
11038
11039	/**
11040	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11041	* re-entry next time around.
11042	*
11043	* @returns Strict VBox status code (i.e. informational status codes too).
11044	* @param pVCpu The cross context virtual CPU structure.
11045	* @param pVmxTransient The VMX-transient structure.
11046	* @param pDbgState The debug state.
11047	* @param rcStrict The return code from executing the guest using single
11048	* stepping.
11049	*/
11050	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11051	VBOXSTRICTRC rcStrict)
11052	{
11053	/*
11054	* Restore VM-exit control settings as we may not reenter this function the
11055	* next time around.
11056	*/
11057	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11058
11059	/* We reload the initial value, trigger what we can of recalculations the
11060	next time around. From the looks of things, that's all that's required atm. */
11061	if (pDbgState->fModifiedProcCtls)
11062	{
11063	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11064	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11065	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11066	AssertRCReturn(rc2, rc2);
11067	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11068	}
11069
11070	/* We're currently the only ones messing with this one, so just restore the
11071	cached value and reload the field. */
11072	if ( pDbgState->fModifiedProcCtls2
11073	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11074	{
11075	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11076	AssertRCReturn(rc2, rc2);
11077	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11078	}
11079
11080	/* If we've modified the exception bitmap, we restore it and trigger
11081	reloading and partial recalculation the next time around. */
11082	if (pDbgState->fModifiedXcptBitmap)
11083	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11084
11085	return rcStrict;
11086	}
11087
11088
11089	/**
11090	* Configures VM-exit controls for current DBGF and DTrace settings.
11091	*
11092	* This updates @a pDbgState and the VMCS execution control fields to reflect
11093	* the necessary VM-exits demanded by DBGF and DTrace.
11094	*
11095	* @param pVCpu The cross context virtual CPU structure.
11096	* @param pVmxTransient The VMX-transient structure. May update
11097	* fUpdatedTscOffsettingAndPreemptTimer.
11098	* @param pDbgState The debug state.
11099	*/
11100	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11101	{
11102	/*
11103	* Take down the dtrace serial number so we can spot changes.
11104	*/
11105	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11106	ASMCompilerBarrier();
11107
11108	/*
11109	* We'll rebuild most of the middle block of data members (holding the
11110	* current settings) as we go along here, so start by clearing it all.
11111	*/
11112	pDbgState->bmXcptExtra = 0;
11113	pDbgState->fCpe1Extra = 0;
11114	pDbgState->fCpe1Unwanted = 0;
11115	pDbgState->fCpe2Extra = 0;
11116	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11117	pDbgState->bmExitsToCheck[i] = 0;
11118
11119	/*
11120	* Software interrupts (INT XXh) - no idea how to trigger these...
11121	*/
11122	PVM pVM = pVCpu->CTX_SUFF(pVM);
11123	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11124	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11125	{
11126	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11127	}
11128
11129	/*
11130	* INT3 breakpoints - triggered by #BP exceptions.
11131	*/
11132	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11133	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11134
11135	/*
11136	* Exception bitmap and XCPT events+probes.
11137	*/
11138	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11139	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11140	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11141
11142	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11143	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11144	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11145	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11146	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11147	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11148	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11149	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11150	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11151	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11152	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11153	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11154	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11155	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11156	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11157	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11158	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11159	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11160
11161	if (pDbgState->bmXcptExtra)
11162	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11163
11164	/*
11165	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11166	*
11167	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11168	* So, when adding/changing/removing please don't forget to update it.
11169	*
11170	* Some of the macros are picking up local variables to save horizontal space,
11171	* (being able to see it in a table is the lesser evil here).
11172	*/
11173	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11174	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11175	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11176	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11177	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11178	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11179	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11180	} else do { } while (0)
11181	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11182	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11183	{ \
11184	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11185	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11186	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11187	} else do { } while (0)
11188	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11189	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11190	{ \
11191	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11192	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11193	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11194	} else do { } while (0)
11195	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11196	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11197	{ \
11198	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11199	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11200	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11201	} else do { } while (0)
11202
11203	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11204	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11205	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11206	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11207	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11208
11209	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11210	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11211	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11212	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11213	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11214	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11215	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11216	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11217	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11218	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11219	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11220	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11221	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11222	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11223	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11224	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11225	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11226	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11227	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11228	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11229	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11230	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11231	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11232	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11233	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11234	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11235	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11236	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11237	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11238	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11239	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11240	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11241	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11242	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11243	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11244	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11245
11246	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11247	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11248	{
11249	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11250	\| CPUMCTX_EXTRN_APIC_TPR);
11251	AssertRC(rc);
11252
11253	#if 0 /** @todo fix me */
11254	pDbgState->fClearCr0Mask = true;
11255	pDbgState->fClearCr4Mask = true;
11256	#endif
11257	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11258	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11259	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11260	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11261	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11262	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11263	require clearing here and in the loop if we start using it. */
11264	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11265	}
11266	else
11267	{
11268	if (pDbgState->fClearCr0Mask)
11269	{
11270	pDbgState->fClearCr0Mask = false;
11271	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11272	}
11273	if (pDbgState->fClearCr4Mask)
11274	{
11275	pDbgState->fClearCr4Mask = false;
11276	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11277	}
11278	}
11279	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11280	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11281
11282	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11283	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11284	{
11285	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11286	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11287	}
11288	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11289	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11290
11291	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11292	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11293	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11294	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11295	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11296	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11297	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11298	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11299	#if 0 /** @todo too slow, fix handler. */
11300	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11301	#endif
11302	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11303
11304	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11305	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11306	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11307	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11308	{
11309	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11310	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11311	}
11312	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11313	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11314	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11315	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11316
11317	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11318	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11319	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11320	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11321	{
11322	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11323	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11324	}
11325	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11326	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11327	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11328	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11329
11330	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11331	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11332	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11333	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11334	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11335	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11336	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11337	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11338	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11339	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11340	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11341	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11342	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11343	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11344	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11345	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11346	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11347	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11348	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11349	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11350	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11351	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11352
11353	#undef IS_EITHER_ENABLED
11354	#undef SET_ONLY_XBM_IF_EITHER_EN
11355	#undef SET_CPE1_XBM_IF_EITHER_EN
11356	#undef SET_CPEU_XBM_IF_EITHER_EN
11357	#undef SET_CPE2_XBM_IF_EITHER_EN
11358
11359	/*
11360	* Sanitize the control stuff.
11361	*/
11362	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11363	if (pDbgState->fCpe2Extra)
11364	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11365	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11366	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11367	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11368	{
11369	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11370	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11371	}
11372
11373	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11374	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11375	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11376	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11377	}
11378
11379
11380	/**
11381	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11382	* appropriate.
11383	*
11384	* The caller has checked the VM-exit against the
11385	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11386	* already, so we don't have to do that either.
11387	*
11388	* @returns Strict VBox status code (i.e. informational status codes too).
11389	* @param pVCpu The cross context virtual CPU structure.
11390	* @param pVmxTransient The VMX-transient structure.
11391	* @param uExitReason The VM-exit reason.
11392	*
11393	* @remarks The name of this function is displayed by dtrace, so keep it short
11394	* and to the point. No longer than 33 chars long, please.
11395	*/
11396	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11397	{
11398	/*
11399	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11400	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11401	*
11402	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11403	* does. Must add/change/remove both places. Same ordering, please.
11404	*
11405	* Added/removed events must also be reflected in the next section
11406	* where we dispatch dtrace events.
11407	*/
11408	bool fDtrace1 = false;
11409	bool fDtrace2 = false;
11410	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11411	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11412	uint32_t uEventArg = 0;
11413	#define SET_EXIT(a_EventSubName) \
11414	do { \
11415	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11416	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11417	} while (0)
11418	#define SET_BOTH(a_EventSubName) \
11419	do { \
11420	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11421	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11422	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11423	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11424	} while (0)
11425	switch (uExitReason)
11426	{
11427	case VMX_EXIT_MTF:
11428	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11429
11430	case VMX_EXIT_XCPT_OR_NMI:
11431	{
11432	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11433	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11434	{
11435	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11436	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11437	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11438	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11439	{
11440	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11441	{
11442	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11443	uEventArg = pVmxTransient->uExitIntErrorCode;
11444	}
11445	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11446	switch (enmEvent1)
11447	{
11448	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11449	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11450	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11451	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11452	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11453	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11454	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11455	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11456	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11457	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11458	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11459	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11460	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11461	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11462	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11463	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11464	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11465	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11466	default: break;
11467	}
11468	}
11469	else
11470	AssertFailed();
11471	break;
11472
11473	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11474	uEventArg = idxVector;
11475	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11476	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11477	break;
11478	}
11479	break;
11480	}
11481
11482	case VMX_EXIT_TRIPLE_FAULT:
11483	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11484	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11485	break;
11486	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11487	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11488	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11489	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11490	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11491
11492	/* Instruction specific VM-exits: */
11493	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11494	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11495	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11496	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11497	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11498	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11499	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11500	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11501	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11502	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11503	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11504	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11505	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11506	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11507	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11508	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11509	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11510	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11511	case VMX_EXIT_MOV_CRX:
11512	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11513	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11514	SET_BOTH(CRX_READ);
11515	else
11516	SET_BOTH(CRX_WRITE);
11517	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11518	break;
11519	case VMX_EXIT_MOV_DRX:
11520	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11521	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11522	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11523	SET_BOTH(DRX_READ);
11524	else
11525	SET_BOTH(DRX_WRITE);
11526	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11527	break;
11528	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11529	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11530	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11531	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11532	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11533	case VMX_EXIT_GDTR_IDTR_ACCESS:
11534	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11535	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11536	{
11537	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11538	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11539	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11540	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11541	}
11542	break;
11543
11544	case VMX_EXIT_LDTR_TR_ACCESS:
11545	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11546	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11547	{
11548	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11549	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11550	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11551	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11552	}
11553	break;
11554
11555	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11556	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11557	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11558	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11559	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11560	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11561	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11562	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11563	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11564	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11565	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11566
11567	/* Events that aren't relevant at this point. */
11568	case VMX_EXIT_EXT_INT:
11569	case VMX_EXIT_INT_WINDOW:
11570	case VMX_EXIT_NMI_WINDOW:
11571	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11572	case VMX_EXIT_PREEMPT_TIMER:
11573	case VMX_EXIT_IO_INSTR:
11574	break;
11575
11576	/* Errors and unexpected events. */
11577	case VMX_EXIT_INIT_SIGNAL:
11578	case VMX_EXIT_SIPI:
11579	case VMX_EXIT_IO_SMI:
11580	case VMX_EXIT_SMI:
11581	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11582	case VMX_EXIT_ERR_MSR_LOAD:
11583	case VMX_EXIT_ERR_MACHINE_CHECK:
11584	break;
11585
11586	default:
11587	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11588	break;
11589	}
11590	#undef SET_BOTH
11591	#undef SET_EXIT
11592
11593	/*
11594	* Dtrace tracepoints go first. We do them here at once so we don't
11595	* have to copy the guest state saving and stuff a few dozen times.
11596	* Down side is that we've got to repeat the switch, though this time
11597	* we use enmEvent since the probes are a subset of what DBGF does.
11598	*/
11599	if (fDtrace1 \|\| fDtrace2)
11600	{
11601	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11602	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11603	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11604	switch (enmEvent1)
11605	{
11606	/** @todo consider which extra parameters would be helpful for each probe. */
11607	case DBGFEVENT_END: break;
11608	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11609	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11610	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11611	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11612	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11613	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11614	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11615	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11616	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
11617	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
11618	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
11619	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
11620	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
11621	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
11622	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
11623	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
11624	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
11625	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
11626	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11627	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11628	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
11629	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
11630	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
11631	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
11632	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
11633	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
11634	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
11635	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11636	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11637	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11638	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11639	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11640	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
11641	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11642	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
11643	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
11644	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
11645	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
11646	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
11647	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
11648	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
11649	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
11650	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
11651	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
11652	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
11653	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
11654	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
11655	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
11656	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
11657	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
11658	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
11659	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
11660	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
11661	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
11662	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
11663	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
11664	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
11665	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
11666	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
11667	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
11668	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
11669	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
11670	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
11671	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
11672	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
11673	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
11674	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
11675	}
11676	switch (enmEvent2)
11677	{
11678	/** @todo consider which extra parameters would be helpful for each probe. */
11679	case DBGFEVENT_END: break;
11680	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
11681	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11682	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
11683	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
11684	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
11685	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
11686	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
11687	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
11688	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
11689	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11690	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11691	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11692	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11693	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11694	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
11695	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11696	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
11697	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
11698	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
11699	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
11700	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
11701	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
11702	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
11703	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
11704	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
11705	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
11706	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
11707	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
11708	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
11709	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
11710	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
11711	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
11712	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
11713	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
11714	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
11715	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
11716	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
11717	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
11718	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
11719	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
11720	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
11721	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
11722	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
11723	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
11724	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
11725	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
11726	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
11727	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
11728	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
11729	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
11730	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
11731	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
11732	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
11733	}
11734	}
11735
11736	/*
11737	* Fire of the DBGF event, if enabled (our check here is just a quick one,
11738	* the DBGF call will do a full check).
11739	*
11740	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
11741	* Note! If we have to events, we prioritize the first, i.e. the instruction
11742	* one, in order to avoid event nesting.
11743	*/
11744	PVM pVM = pVCpu->CTX_SUFF(pVM);
11745	if ( enmEvent1 != DBGFEVENT_END
11746	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
11747	{
11748	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11749	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
11750	if (rcStrict != VINF_SUCCESS)
11751	return rcStrict;
11752	}
11753	else if ( enmEvent2 != DBGFEVENT_END
11754	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
11755	{
11756	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11757	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
11758	if (rcStrict != VINF_SUCCESS)
11759	return rcStrict;
11760	}
11761
11762	return VINF_SUCCESS;
11763	}
11764
11765
11766	/**
11767	* Single-stepping VM-exit filtering.
11768	*
11769	* This is preprocessing the VM-exits and deciding whether we've gotten far
11770	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
11771	* handling is performed.
11772	*
11773	* @returns Strict VBox status code (i.e. informational status codes too).
11774	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
11775	* @param pVmxTransient The VMX-transient structure.
11776	* @param pDbgState The debug state.
11777	*/
11778	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11779	{
11780	/*
11781	* Expensive (saves context) generic dtrace VM-exit probe.
11782	*/
11783	uint32_t const uExitReason = pVmxTransient->uExitReason;
11784	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
11785	{ /* more likely */ }
11786	else
11787	{
11788	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11789	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11790	AssertRC(rc);
11791	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
11792	}
11793
11794	/*
11795	* Check for host NMI, just to get that out of the way.
11796	*/
11797	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
11798	{ /* normally likely */ }
11799	else
11800	{
11801	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11802	AssertRCReturn(rc2, rc2);
11803	uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11804	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11805	return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
11806	}
11807
11808	/*
11809	* Check for single stepping event if we're stepping.
11810	*/
11811	if (pVCpu->hm.s.fSingleInstruction)
11812	{
11813	switch (uExitReason)
11814	{
11815	case VMX_EXIT_MTF:
11816	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11817
11818	/* Various events: */
11819	case VMX_EXIT_XCPT_OR_NMI:
11820	case VMX_EXIT_EXT_INT:
11821	case VMX_EXIT_TRIPLE_FAULT:
11822	case VMX_EXIT_INT_WINDOW:
11823	case VMX_EXIT_NMI_WINDOW:
11824	case VMX_EXIT_TASK_SWITCH:
11825	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11826	case VMX_EXIT_APIC_ACCESS:
11827	case VMX_EXIT_EPT_VIOLATION:
11828	case VMX_EXIT_EPT_MISCONFIG:
11829	case VMX_EXIT_PREEMPT_TIMER:
11830
11831	/* Instruction specific VM-exits: */
11832	case VMX_EXIT_CPUID:
11833	case VMX_EXIT_GETSEC:
11834	case VMX_EXIT_HLT:
11835	case VMX_EXIT_INVD:
11836	case VMX_EXIT_INVLPG:
11837	case VMX_EXIT_RDPMC:
11838	case VMX_EXIT_RDTSC:
11839	case VMX_EXIT_RSM:
11840	case VMX_EXIT_VMCALL:
11841	case VMX_EXIT_VMCLEAR:
11842	case VMX_EXIT_VMLAUNCH:
11843	case VMX_EXIT_VMPTRLD:
11844	case VMX_EXIT_VMPTRST:
11845	case VMX_EXIT_VMREAD:
11846	case VMX_EXIT_VMRESUME:
11847	case VMX_EXIT_VMWRITE:
11848	case VMX_EXIT_VMXOFF:
11849	case VMX_EXIT_VMXON:
11850	case VMX_EXIT_MOV_CRX:
11851	case VMX_EXIT_MOV_DRX:
11852	case VMX_EXIT_IO_INSTR:
11853	case VMX_EXIT_RDMSR:
11854	case VMX_EXIT_WRMSR:
11855	case VMX_EXIT_MWAIT:
11856	case VMX_EXIT_MONITOR:
11857	case VMX_EXIT_PAUSE:
11858	case VMX_EXIT_GDTR_IDTR_ACCESS:
11859	case VMX_EXIT_LDTR_TR_ACCESS:
11860	case VMX_EXIT_INVEPT:
11861	case VMX_EXIT_RDTSCP:
11862	case VMX_EXIT_INVVPID:
11863	case VMX_EXIT_WBINVD:
11864	case VMX_EXIT_XSETBV:
11865	case VMX_EXIT_RDRAND:
11866	case VMX_EXIT_INVPCID:
11867	case VMX_EXIT_VMFUNC:
11868	case VMX_EXIT_RDSEED:
11869	case VMX_EXIT_XSAVES:
11870	case VMX_EXIT_XRSTORS:
11871	{
11872	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11873	AssertRCReturn(rc, rc);
11874	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
11875	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
11876	return VINF_EM_DBG_STEPPED;
11877	break;
11878	}
11879
11880	/* Errors and unexpected events: */
11881	case VMX_EXIT_INIT_SIGNAL:
11882	case VMX_EXIT_SIPI:
11883	case VMX_EXIT_IO_SMI:
11884	case VMX_EXIT_SMI:
11885	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11886	case VMX_EXIT_ERR_MSR_LOAD:
11887	case VMX_EXIT_ERR_MACHINE_CHECK:
11888	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
11889	break;
11890
11891	default:
11892	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11893	break;
11894	}
11895	}
11896
11897	/*
11898	* Check for debugger event breakpoints and dtrace probes.
11899	*/
11900	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
11901	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
11902	{
11903	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
11904	if (rcStrict != VINF_SUCCESS)
11905	return rcStrict;
11906	}
11907
11908	/*
11909	* Normal processing.
11910	*/
11911	#ifdef HMVMX_USE_FUNCTION_TABLE
11912	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
11913	#else
11914	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
11915	#endif
11916	}
11917
11918
11919	/**
11920	* Single steps guest code using hardware-assisted VMX.
11921	*
11922	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
11923	* but single-stepping through the hypervisor debugger.
11924	*
11925	* @returns Strict VBox status code (i.e. informational status codes too).
11926	* @param pVCpu The cross context virtual CPU structure.
11927	* @param pcLoops Pointer to the number of executed loops.
11928	*
11929	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
11930	*/
11931	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
11932	{
11933	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11934	Assert(pcLoops);
11935	Assert(*pcLoops <= cMaxResumeLoops);
11936
11937	VMXTRANSIENT VmxTransient;
11938	RT_ZERO(VmxTransient);
11939	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11940
11941	/* Set HMCPU indicators. */
11942	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
11943	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
11944	pVCpu->hm.s.fDebugWantRdTscExit = false;
11945	pVCpu->hm.s.fUsingDebugLoop = true;
11946
11947	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
11948	VMXRUNDBGSTATE DbgState;
11949	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
11950	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
11951
11952	/*
11953	* The loop.
11954	*/
11955	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11956	for (;;)
11957	{
11958	Assert(!HMR0SuspendPending());
11959	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11960	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11961	bool fStepping = pVCpu->hm.s.fSingleInstruction;
11962
11963	/* Set up VM-execution controls the next two can respond to. */
11964	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
11965
11966	/*
11967	* Preparatory work for running guest code, this may force us to
11968	* return to ring-3.
11969	*
11970	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11971	*/
11972	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
11973	if (rcStrict != VINF_SUCCESS)
11974	break;
11975
11976	/* Interrupts are disabled at this point! */
11977	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11978
11979	/* Override any obnoxious code in the above two calls. */
11980	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
11981
11982	/*
11983	* Finally execute the guest.
11984	*/
11985	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11986
11987	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11988	/* Interrupts are re-enabled at this point! */
11989
11990	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
11991	if (RT_SUCCESS(rcRun))
11992	{ /* very likely */ }
11993	else
11994	{
11995	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11996	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11997	return rcRun;
11998	}
11999
12000	/* Profile the VM-exit. */
12001	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12002	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12003	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12004	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12005	HMVMX_START_EXIT_DISPATCH_PROF();
12006
12007	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12008
12009	/*
12010	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12011	*/
12012	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12013	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12014	if (rcStrict != VINF_SUCCESS)
12015	break;
12016	if (++(*pcLoops) > cMaxResumeLoops)
12017	{
12018	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12019	rcStrict = VINF_EM_RAW_INTERRUPT;
12020	break;
12021	}
12022
12023	/*
12024	* Stepping: Did the RIP change, if so, consider it a single step.
12025	* Otherwise, make sure one of the TFs gets set.
12026	*/
12027	if (fStepping)
12028	{
12029	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12030	AssertRC(rc);
12031	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12032	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12033	{
12034	rcStrict = VINF_EM_DBG_STEPPED;
12035	break;
12036	}
12037	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12038	}
12039
12040	/*
12041	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12042	*/
12043	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12044	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12045	}
12046
12047	/*
12048	* Clear the X86_EFL_TF if necessary.
12049	*/
12050	if (pVCpu->hm.s.fClearTrapFlag)
12051	{
12052	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12053	AssertRC(rc);
12054	pVCpu->hm.s.fClearTrapFlag = false;
12055	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12056	}
12057	/** @todo there seems to be issues with the resume flag when the monitor trap
12058	* flag is pending without being used. Seen early in bios init when
12059	* accessing APIC page in protected mode. */
12060
12061	/*
12062	* Restore VM-exit control settings as we may not re-enter this function the
12063	* next time around.
12064	*/
12065	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12066
12067	/* Restore HMCPU indicators. */
12068	pVCpu->hm.s.fUsingDebugLoop = false;
12069	pVCpu->hm.s.fDebugWantRdTscExit = false;
12070	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12071
12072	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12073	return rcStrict;
12074	}
12075
12076
12077	/** @} */
12078
12079
12080	/**
12081	* Checks if any expensive dtrace probes are enabled and we should go to the
12082	* debug loop.
12083	*
12084	* @returns true if we should use debug loop, false if not.
12085	*/
12086	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12087	{
12088	/* It's probably faster to OR the raw 32-bit counter variables together.
12089	Since the variables are in an array and the probes are next to one
12090	another (more or less), we have good locality. So, better read
12091	eight-nine cache lines ever time and only have one conditional, than
12092	128+ conditionals, right? */
12093	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12094	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12095	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12096	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12097	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12098	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12099	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12100	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12101	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12102	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12103	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12104	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12105	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12106	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12107	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12108	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12109	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12110	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12111	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12112	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12113	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12114	) != 0
12115	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12116	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12117	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12118	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12119	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12120	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12121	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12122	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12123	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12124	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12125	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12126	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12127	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12128	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12129	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12130	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12131	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12132	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12133	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12134	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12135	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12136	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12137	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12138	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12139	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12140	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12141	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12142	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12143	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12144	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12145	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12146	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12147	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12148	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12149	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12150	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12151	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12152	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12153	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12154	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12155	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12156	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12157	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12158	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12159	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12160	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12161	) != 0
12162	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12163	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12164	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12165	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12166	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12167	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12168	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12169	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12170	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12171	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12172	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12173	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12174	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12175	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12176	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12177	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12178	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12179	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12180	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12181	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12182	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12183	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12184	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12185	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12186	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12187	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12188	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12189	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12190	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12191	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12192	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12193	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12194	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12195	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12196	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12197	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12198	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12199	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12200	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12201	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12202	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12203	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12204	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12205	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12206	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12207	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12208	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12209	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12210	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12211	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12212	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12213	) != 0;
12214	}
12215
12216
12217	/**
12218	* Runs the guest using hardware-assisted VMX.
12219	*
12220	* @returns Strict VBox status code (i.e. informational status codes too).
12221	* @param pVCpu The cross context virtual CPU structure.
12222	*/
12223	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
12224	{
12225	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12226	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12227	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12228	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12229
12230	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
12231
12232	VBOXSTRICTRC rcStrict;
12233	uint32_t cLoops = 0;
12234	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12235	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12236	#else
12237	bool const fInNestedGuestMode = false;
12238	#endif
12239	if (!fInNestedGuestMode)
12240	{
12241	if ( !pVCpu->hm.s.fUseDebugLoop
12242	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12243	&& !DBGFIsStepping(pVCpu)
12244	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12245	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12246	else
12247	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12248	}
12249	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12250	else
12251	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
12252
12253	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12254	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12255	#endif
12256
12257	if (rcStrict == VERR_EM_INTERPRETER)
12258	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12259	else if (rcStrict == VINF_EM_RESET)
12260	rcStrict = VINF_EM_TRIPLE_FAULT;
12261
12262	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12263	if (RT_FAILURE(rc2))
12264	{
12265	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12266	rcStrict = rc2;
12267	}
12268	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12269	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12270	return rcStrict;
12271	}
12272
12273
12274	#ifndef HMVMX_USE_FUNCTION_TABLE
12275	/**
12276	* Handles a guest VM-exit from hardware-assisted VMX execution.
12277	*
12278	* @returns Strict VBox status code (i.e. informational status codes too).
12279	* @param pVCpu The cross context virtual CPU structure.
12280	* @param pVmxTransient The VMX-transient structure.
12281	*/
12282	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12283	{
12284	#ifdef DEBUG_ramshankar
12285	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12286	do { \
12287	if (a_fSave != 0) \
12288	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
12289	VBOXSTRICTRC rcStrict = a_CallExpr; \
12290	if (a_fSave != 0) \
12291	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12292	return rcStrict; \
12293	} while (0)
12294	#else
12295	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12296	#endif
12297	uint32_t const rcReason = pVmxTransient->uExitReason;
12298	switch (rcReason)
12299	{
12300	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12301	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12302	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12303	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12304	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12305	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12306	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12307	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12308	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12309	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12310	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12311	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12312	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12313	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12314	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12315	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12316	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12317	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12318	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12319	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12320	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12321	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12322	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12323	case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
12324	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12325	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12326	case VMX_EXIT_GDTR_IDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12327	case VMX_EXIT_LDTR_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12328	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12329	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12330	case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
12331	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12332	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12333	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12334	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12335	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12336	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12337	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12338	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12339	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12340	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12341	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12342	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12343	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12344	#else
12345	case VMX_EXIT_VMCLEAR:
12346	case VMX_EXIT_VMLAUNCH:
12347	case VMX_EXIT_VMPTRLD:
12348	case VMX_EXIT_VMPTRST:
12349	case VMX_EXIT_VMREAD:
12350	case VMX_EXIT_VMRESUME:
12351	case VMX_EXIT_VMWRITE:
12352	case VMX_EXIT_VMXOFF:
12353	case VMX_EXIT_VMXON:
12354	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12355	#endif
12356
12357	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12358	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12359	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
12360	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
12361	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
12362	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
12363	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
12364	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12365	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
12366
12367	case VMX_EXIT_INVEPT:
12368	case VMX_EXIT_INVVPID:
12369	case VMX_EXIT_VMFUNC:
12370	case VMX_EXIT_XSAVES:
12371	case VMX_EXIT_XRSTORS:
12372	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12373
12374	case VMX_EXIT_ENCLS:
12375	case VMX_EXIT_RDSEED:
12376	case VMX_EXIT_PML_FULL:
12377	default:
12378	return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
12379	}
12380	#undef VMEXIT_CALL_RET
12381	}
12382	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12383
12384
12385	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12386	/**
12387	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12388	*
12389	* @returns Strict VBox status code (i.e. informational status codes too).
12390	* @param pVCpu The cross context virtual CPU structure.
12391	* @param pVmxTransient The VMX-transient structure.
12392	*/
12393	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12394	{
12395	uint32_t const rcReason = pVmxTransient->uExitReason;
12396	switch (rcReason)
12397	{
12398	case VMX_EXIT_EPT_MISCONFIG:
12399	case VMX_EXIT_EPT_VIOLATION:
12400	case VMX_EXIT_IO_INSTR:
12401	case VMX_EXIT_CPUID:
12402	case VMX_EXIT_RDTSC:
12403	case VMX_EXIT_RDTSCP:
12404	case VMX_EXIT_APIC_ACCESS:
12405	case VMX_EXIT_XCPT_OR_NMI:
12406	case VMX_EXIT_MOV_CRX:
12407	case VMX_EXIT_EXT_INT:
12408	case VMX_EXIT_INT_WINDOW:
12409	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12410	case VMX_EXIT_MWAIT:
12411	case VMX_EXIT_MONITOR:
12412	case VMX_EXIT_TASK_SWITCH:
12413	case VMX_EXIT_PREEMPT_TIMER:
12414	case VMX_EXIT_RDMSR:
12415	case VMX_EXIT_WRMSR:
12416	case VMX_EXIT_VMCALL:
12417	case VMX_EXIT_MOV_DRX:
12418	case VMX_EXIT_HLT:
12419	case VMX_EXIT_INVD:
12420	case VMX_EXIT_INVLPG:
12421	case VMX_EXIT_RSM:
12422	case VMX_EXIT_MTF:
12423	case VMX_EXIT_PAUSE:
12424	case VMX_EXIT_GDTR_IDTR_ACCESS:
12425	case VMX_EXIT_LDTR_TR_ACCESS:
12426	case VMX_EXIT_WBINVD:
12427	case VMX_EXIT_XSETBV:
12428	case VMX_EXIT_RDRAND:
12429	case VMX_EXIT_INVPCID:
12430	case VMX_EXIT_GETSEC:
12431	case VMX_EXIT_RDPMC:
12432	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12433	case VMX_EXIT_VMCLEAR:
12434	case VMX_EXIT_VMLAUNCH:
12435	case VMX_EXIT_VMPTRLD:
12436	case VMX_EXIT_VMPTRST:
12437	case VMX_EXIT_VMREAD:
12438	case VMX_EXIT_VMRESUME:
12439	case VMX_EXIT_VMWRITE:
12440	case VMX_EXIT_VMXOFF:
12441	case VMX_EXIT_VMXON:
12442	#endif
12443	case VMX_EXIT_TRIPLE_FAULT:
12444	case VMX_EXIT_NMI_WINDOW:
12445	case VMX_EXIT_INIT_SIGNAL:
12446	case VMX_EXIT_SIPI:
12447	case VMX_EXIT_IO_SMI:
12448	case VMX_EXIT_SMI:
12449	case VMX_EXIT_ERR_MSR_LOAD:
12450	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12451	case VMX_EXIT_ERR_MACHINE_CHECK:
12452
12453	case VMX_EXIT_INVEPT:
12454	case VMX_EXIT_INVVPID:
12455	case VMX_EXIT_VMFUNC:
12456	case VMX_EXIT_XSAVES:
12457	case VMX_EXIT_XRSTORS:
12458
12459	case VMX_EXIT_ENCLS:
12460	case VMX_EXIT_RDSEED:
12461	case VMX_EXIT_PML_FULL:
12462	default:
12463	return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
12464	}
12465	#undef VMEXIT_CALL_RET
12466	}
12467	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12468
12469
12470	#ifdef VBOX_STRICT
12471	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
12472	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
12473	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
12474
12475	# define HMVMX_ASSERT_PREEMPT_CPUID() \
12476	do { \
12477	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
12478	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
12479	} while (0)
12480
12481	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12482	do { \
12483	AssertPtr((a_pVCpu)); \
12484	AssertPtr((a_pVmxTransient)); \
12485	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
12486	Assert((a_pVmxTransient)->pVmcsInfo); \
12487	Assert(ASMIntAreEnabled()); \
12488	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12489	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
12490	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
12491	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12492	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
12493	HMVMX_ASSERT_PREEMPT_CPUID(); \
12494	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12495	} while (0)
12496
12497	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12498	do { \
12499	Log4Func(("\n")); \
12500	} while (0)
12501	#else
12502	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12503	do { \
12504	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12505	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
12506	} while (0)
12507	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
12508	#endif
12509
12510
12511	/**
12512	* Advances the guest RIP by the specified number of bytes.
12513	*
12514	* @param pVCpu The cross context virtual CPU structure.
12515	* @param cbInstr Number of bytes to advance the RIP by.
12516	*
12517	* @remarks No-long-jump zone!!!
12518	*/
12519	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
12520	{
12521	/* Advance the RIP. */
12522	pVCpu->cpum.GstCtx.rip += cbInstr;
12523	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12524
12525	/* Update interrupt inhibition. */
12526	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
12527	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
12528	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12529	}
12530
12531
12532	/**
12533	* Advances the guest RIP after reading it from the VMCS.
12534	*
12535	* @returns VBox status code, no informational status codes.
12536	* @param pVCpu The cross context virtual CPU structure.
12537	* @param pVmxTransient The VMX-transient structure.
12538	*
12539	* @remarks No-long-jump zone!!!
12540	*/
12541	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12542	{
12543	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12544	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
12545	AssertRCReturn(rc, rc);
12546
12547	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
12548	return VINF_SUCCESS;
12549	}
12550
12551
12552	/**
12553	* Handle a condition that occurred while delivering an event through the guest
12554	* IDT.
12555	*
12556	* @returns Strict VBox status code (i.e. informational status codes too).
12557	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
12558	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
12559	* to continue execution of the guest which will delivery the \#DF.
12560	* @retval VINF_EM_RESET if we detected a triple-fault condition.
12561	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
12562	*
12563	* @param pVCpu The cross context virtual CPU structure.
12564	* @param pVmxTransient The VMX-transient structure.
12565	*
12566	* @remarks No-long-jump zone!!!
12567	*/
12568	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12569	{
12570	uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12571
12572	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12573	rc2 \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12574	AssertRCReturn(rc2, rc2);
12575
12576	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
12577	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12578	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12579	{
12580	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12581	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12582
12583	/*
12584	* If the event was a software interrupt (generated with INT n) or a software exception
12585	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
12586	* can handle the VM-exit and continue guest execution which will re-execute the
12587	* instruction rather than re-injecting the exception, as that can cause premature
12588	* trips to ring-3 before injection and involve TRPM which currently has no way of
12589	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
12590	* the problem).
12591	*/
12592	IEMXCPTRAISE enmRaise;
12593	IEMXCPTRAISEINFO fRaiseInfo;
12594	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
12595	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
12596	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
12597	{
12598	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
12599	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12600	}
12601	else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
12602	{
12603	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12604	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
12605	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
12606	/** @todo Make AssertMsgReturn as just AssertMsg later. */
12607	AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
12608	("Unexpected VM-exit interruption vector type %#x!\n", uExitVectorType), VERR_VMX_IPE_5);
12609
12610	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
12611
12612	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
12613	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
12614	{
12615	pVmxTransient->fVectoringPF = true;
12616	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12617	}
12618	}
12619	else
12620	{
12621	/*
12622	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
12623	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
12624	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
12625	*/
12626	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12627	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12628	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
12629	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12630	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12631	}
12632
12633	/*
12634	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
12635	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
12636	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
12637	* subsequent VM-entry would fail.
12638	*
12639	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
12640	*/
12641	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS)
12642	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12643	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
12644	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
12645	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
12646	{
12647	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
12648	}
12649
12650	switch (enmRaise)
12651	{
12652	case IEMXCPTRAISE_CURRENT_XCPT:
12653	{
12654	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
12655	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
12656	Assert(rcStrict == VINF_SUCCESS);
12657	break;
12658	}
12659
12660	case IEMXCPTRAISE_PREV_EVENT:
12661	{
12662	uint32_t u32ErrCode;
12663	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
12664	{
12665	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12666	AssertRCReturn(rc2, rc2);
12667	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12668	}
12669	else
12670	u32ErrCode = 0;
12671
12672	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
12673	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
12674	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12675	0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
12676
12677	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
12678	pVCpu->hm.s.Event.u32ErrCode));
12679	Assert(rcStrict == VINF_SUCCESS);
12680	break;
12681	}
12682
12683	case IEMXCPTRAISE_REEXEC_INSTR:
12684	Assert(rcStrict == VINF_SUCCESS);
12685	break;
12686
12687	case IEMXCPTRAISE_DOUBLE_FAULT:
12688	{
12689	/*
12690	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
12691	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
12692	*/
12693	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
12694	{
12695	pVmxTransient->fVectoringDoublePF = true;
12696	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
12697	pVCpu->cpum.GstCtx.cr2));
12698	rcStrict = VINF_SUCCESS;
12699	}
12700	else
12701	{
12702	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
12703	hmR0VmxSetPendingXcptDF(pVCpu);
12704	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
12705	uIdtVector, uExitVector));
12706	rcStrict = VINF_HM_DOUBLE_FAULT;
12707	}
12708	break;
12709	}
12710
12711	case IEMXCPTRAISE_TRIPLE_FAULT:
12712	{
12713	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
12714	rcStrict = VINF_EM_RESET;
12715	break;
12716	}
12717
12718	case IEMXCPTRAISE_CPU_HANG:
12719	{
12720	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
12721	rcStrict = VERR_EM_GUEST_CPU_HANG;
12722	break;
12723	}
12724
12725	default:
12726	{
12727	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
12728	rcStrict = VERR_VMX_IPE_2;
12729	break;
12730	}
12731	}
12732	}
12733	else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
12734	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
12735	&& uExitVector != X86_XCPT_DF
12736	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
12737	{
12738	/*
12739	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
12740	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
12741	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
12742	*/
12743	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
12744	{
12745	Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
12746	VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
12747	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
12748	}
12749	}
12750
12751	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
12752	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
12753	return rcStrict;
12754	}
12755
12756
12757	/** @name VM-exit handlers.
12758	* @{
12759	*/
12760	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12761	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12762	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12763
12764	/**
12765	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
12766	*/
12767	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12768	{
12769	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12770	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
12771	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
12772	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
12773	return VINF_SUCCESS;
12774	return VINF_EM_RAW_INTERRUPT;
12775	}
12776
12777
12778	/**
12779	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
12780	*/
12781	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12782	{
12783	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12784	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
12785
12786	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12787	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12788	AssertRCReturn(rc, rc);
12789
12790	uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12791	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
12792	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
12793	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
12794
12795	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12796	{
12797	/*
12798	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
12799	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
12800	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
12801	*
12802	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
12803	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
12804	*/
12805	VMXDispatchHostNmi();
12806	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
12807	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12808	return VINF_SUCCESS;
12809	}
12810
12811	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12812	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12813	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
12814	{ /* likely */ }
12815	else
12816	{
12817	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
12818	rcStrictRc1 = VINF_SUCCESS;
12819	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12820	return rcStrictRc1;
12821	}
12822
12823	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
12824	uint32_t uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
12825	switch (uIntType)
12826	{
12827	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
12828	Assert(uVector == X86_XCPT_DB);
12829	RT_FALL_THRU();
12830	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
12831	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
12832	RT_FALL_THRU();
12833	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
12834	{
12835	/*
12836	* If there's any exception caused as a result of event injection, the resulting
12837	* secondary/final execption will be pending, we shall continue guest execution
12838	* after injecting the event. The page-fault case is complicated and we manually
12839	* handle any currently pending event in hmR0VmxExitXcptPF.
12840	*/
12841	if (!pVCpu->hm.s.Event.fPending)
12842	{ /* likely */ }
12843	else if (uVector != X86_XCPT_PF)
12844	{
12845	rc = VINF_SUCCESS;
12846	break;
12847	}
12848
12849	switch (uVector)
12850	{
12851	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
12852	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
12853	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
12854	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
12855	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
12856	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
12857
12858	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
12859	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12860	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
12861	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12862	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
12863	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12864	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
12865	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12866	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
12867	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12868	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
12869	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12870	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
12871	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12872	default:
12873	{
12874	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
12875	if (pVmcsInfo->RealMode.fRealOnV86Active)
12876	{
12877	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
12878	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
12879	Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
12880
12881	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
12882	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12883	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12884	AssertRCReturn(rc, rc);
12885	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
12886	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
12887	0 /* GCPtrFaultAddress */);
12888	}
12889	else
12890	{
12891	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
12892	pVCpu->hm.s.u32HMError = uVector;
12893	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
12894	}
12895	break;
12896	}
12897	}
12898	break;
12899	}
12900
12901	default:
12902	{
12903	pVCpu->hm.s.u32HMError = uExitIntInfo;
12904	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
12905	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
12906	break;
12907	}
12908	}
12909	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12910	return rc;
12911	}
12912
12913
12914	/**
12915	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
12916	*/
12917	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12918	{
12919	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12920
12921	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
12922	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12923	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
12924	AssertRCReturn(rc, rc);
12925
12926	/* Evaluate and deliver pending events and resume guest execution. */
12927	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
12928	return VINF_SUCCESS;
12929	}
12930
12931
12932	/**
12933	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
12934	*/
12935	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12936	{
12937	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12938
12939	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12940	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
12941	{
12942	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
12943	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12944	}
12945
12946	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS));
12947
12948	/*
12949	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
12950	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
12951	*/
12952	uint32_t fIntrState;
12953	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
12954	AssertRCReturn(rc, rc);
12955	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
12956	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
12957	{
12958	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
12959	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12960
12961	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
12962	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
12963	AssertRCReturn(rc, rc);
12964	}
12965
12966	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
12967	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
12968	AssertRCReturn(rc, rc);
12969
12970	/* Evaluate and deliver pending events and resume guest execution. */
12971	return VINF_SUCCESS;
12972	}
12973
12974
12975	/**
12976	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
12977	*/
12978	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12979	{
12980	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12981	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12982	}
12983
12984
12985	/**
12986	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
12987	*/
12988	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12989	{
12990	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12991	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12992	}
12993
12994
12995	/**
12996	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
12997	*/
12998	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12999	{
13000	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13001
13002	/*
13003	* Get the state we need and update the exit history entry.
13004	*/
13005	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13006	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13007	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13008	AssertRCReturn(rc, rc);
13009
13010	VBOXSTRICTRC rcStrict;
13011	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
13012	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
13013	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
13014	if (!pExitRec)
13015	{
13016	/*
13017	* Regular CPUID instruction execution.
13018	*/
13019	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
13020	if (rcStrict == VINF_SUCCESS)
13021	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13022	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13023	{
13024	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13025	rcStrict = VINF_SUCCESS;
13026	}
13027	}
13028	else
13029	{
13030	/*
13031	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
13032	*/
13033	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13034	AssertRCReturn(rc2, rc2);
13035
13036	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
13037	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
13038
13039	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
13040	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13041
13042	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
13043	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13044	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
13045	}
13046	return rcStrict;
13047	}
13048
13049
13050	/**
13051	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
13052	*/
13053	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13054	{
13055	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13056
13057	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13058	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
13059	AssertRCReturn(rc, rc);
13060
13061	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
13062	return VINF_EM_RAW_EMULATE_INSTR;
13063
13064	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
13065	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13066	}
13067
13068
13069	/**
13070	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
13071	*/
13072	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13073	{
13074	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13075
13076	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13077	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13078	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13079	AssertRCReturn(rc, rc);
13080
13081	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
13082	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13083	{
13084	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13085	we must reset offsetting on VM-entry. See @bugref{6634}. */
13086	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13087	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13088	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13089	}
13090	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13091	{
13092	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13093	rcStrict = VINF_SUCCESS;
13094	}
13095	return rcStrict;
13096	}
13097
13098
13099	/**
13100	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
13101	*/
13102	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13103	{
13104	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13105
13106	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13107	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
13108	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13109	AssertRCReturn(rc, rc);
13110
13111	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
13112	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13113	{
13114	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13115	we must reset offsetting on VM-reentry. See @bugref{6634}. */
13116	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13117	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13118	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13119	}
13120	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13121	{
13122	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13123	rcStrict = VINF_SUCCESS;
13124	}
13125	return rcStrict;
13126	}
13127
13128
13129	/**
13130	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
13131	*/
13132	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13133	{
13134	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13135
13136	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13137	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
13138	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13139	AssertRCReturn(rc, rc);
13140
13141	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13142	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13143	if (RT_LIKELY(rc == VINF_SUCCESS))
13144	{
13145	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13146	Assert(pVmxTransient->cbInstr == 2);
13147	}
13148	else
13149	{
13150	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
13151	rc = VERR_EM_INTERPRETER;
13152	}
13153	return rc;
13154	}
13155
13156
13157	/**
13158	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
13159	*/
13160	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13161	{
13162	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13163
13164	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
13165	if (EMAreHypercallInstructionsEnabled(pVCpu))
13166	{
13167	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13168	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
13169	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13170	AssertRCReturn(rc, rc);
13171
13172	/* Perform the hypercall. */
13173	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
13174	if (rcStrict == VINF_SUCCESS)
13175	{
13176	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13177	AssertRCReturn(rc, rc);
13178	}
13179	else
13180	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
13181	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
13182	\|\| RT_FAILURE(rcStrict));
13183
13184	/* If the hypercall changes anything other than guest's general-purpose registers,
13185	we would need to reload the guest changed bits here before VM-entry. */
13186	}
13187	else
13188	Log4Func(("Hypercalls not enabled\n"));
13189
13190	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
13191	if (RT_FAILURE(rcStrict))
13192	{
13193	hmR0VmxSetPendingXcptUD(pVCpu);
13194	rcStrict = VINF_SUCCESS;
13195	}
13196
13197	return rcStrict;
13198	}
13199
13200
13201	/**
13202	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
13203	*/
13204	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13205	{
13206	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13207	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
13208
13209	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13210	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13211	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13212	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13213	AssertRCReturn(rc, rc);
13214
13215	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
13216
13217	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
13218	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13219	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13220	{
13221	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13222	rcStrict = VINF_SUCCESS;
13223	}
13224	else
13225	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n", pVmxTransient->uExitQual,
13226	VBOXSTRICTRC_VAL(rcStrict)));
13227	return rcStrict;
13228	}
13229
13230
13231	/**
13232	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
13233	*/
13234	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13235	{
13236	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13237
13238	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13239	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13240	AssertRCReturn(rc, rc);
13241
13242	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13243	rc = EMInterpretMonitor(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13244	if (RT_LIKELY(rc == VINF_SUCCESS))
13245	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13246	else
13247	{
13248	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
13249	rc = VERR_EM_INTERPRETER;
13250	}
13251	return rc;
13252	}
13253
13254
13255	/**
13256	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
13257	*/
13258	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13259	{
13260	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13261
13262	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13263	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13264	AssertRCReturn(rc, rc);
13265
13266	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13267	VBOXSTRICTRC rc2 = EMInterpretMWait(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13268	rc = VBOXSTRICTRC_VAL(rc2);
13269	if (RT_LIKELY( rc == VINF_SUCCESS
13270	\|\| rc == VINF_EM_HALT))
13271	{
13272	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13273	AssertRCReturn(rc3, rc3);
13274
13275	if ( rc == VINF_EM_HALT
13276	&& EMMonitorWaitShouldContinue(pVCpu, pCtx))
13277	rc = VINF_SUCCESS;
13278	}
13279	else
13280	{
13281	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
13282	rc = VERR_EM_INTERPRETER;
13283	}
13284	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
13285	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
13286	return rc;
13287	}
13288
13289
13290	/**
13291	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
13292	*/
13293	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13294	{
13295	/*
13296	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
13297	* mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
13298	* treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
13299	* VMX root operation. If we get here, something funny is going on.
13300	*
13301	* See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
13302	*/
13303	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13304	AssertMsgFailed(("Unexpected RSM VM-exit\n"));
13305	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13306	}
13307
13308
13309	/**
13310	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
13311	*/
13312	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13313	{
13314	/*
13315	* This can only happen if we support dual-monitor treatment of SMI, which can be activated
13316	* by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
13317	* this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
13318	* an SMI. If we get here, something funny is going on.
13319	*
13320	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
13321	* See Intel spec. 25.3 "Other Causes of VM-Exits"
13322	*/
13323	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13324	AssertMsgFailed(("Unexpected SMI VM-exit\n"));
13325	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13326	}
13327
13328
13329	/**
13330	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
13331	*/
13332	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13333	{
13334	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
13335	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13336	AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
13337	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13338	}
13339
13340
13341	/**
13342	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
13343	*/
13344	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13345	{
13346	/*
13347	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
13348	* We don't make use of it as our guests don't have direct access to the host LAPIC.
13349	* See Intel spec. 25.3 "Other Causes of VM-exits".
13350	*/
13351	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13352	AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
13353	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13354	}
13355
13356
13357	/**
13358	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
13359	* VM-exit.
13360	*/
13361	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13362	{
13363	/*
13364	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
13365	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
13366	*
13367	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
13368	* See Intel spec. "23.8 Restrictions on VMX operation".
13369	*/
13370	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13371	return VINF_SUCCESS;
13372	}
13373
13374
13375	/**
13376	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
13377	* VM-exit.
13378	*/
13379	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13380	{
13381	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13382	return VINF_EM_RESET;
13383	}
13384
13385
13386	/**
13387	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
13388	*/
13389	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13390	{
13391	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13392
13393	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13394	AssertRCReturn(rc, rc);
13395
13396	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
13397	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
13398	rc = VINF_SUCCESS;
13399	else
13400	rc = VINF_EM_HALT;
13401
13402	if (rc != VINF_SUCCESS)
13403	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
13404	return rc;
13405	}
13406
13407
13408	/**
13409	* VM-exit handler for instructions that result in a \#UD exception delivered to
13410	* the guest.
13411	*/
13412	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13413	{
13414	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13415	hmR0VmxSetPendingXcptUD(pVCpu);
13416	return VINF_SUCCESS;
13417	}
13418
13419
13420	/**
13421	* VM-exit handler for expiry of the VMX-preemption timer.
13422	*/
13423	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13424	{
13425	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13426
13427	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
13428	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13429
13430	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
13431	PVM pVM = pVCpu->CTX_SUFF(pVM);
13432	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
13433	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
13434	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
13435	}
13436
13437
13438	/**
13439	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
13440	*/
13441	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13442	{
13443	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13444
13445	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13446	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13447	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
13448	AssertRCReturn(rc, rc);
13449
13450	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
13451	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13452	: HM_CHANGED_RAISED_XCPT_MASK);
13453
13454	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13455	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
13456
13457	return rcStrict;
13458	}
13459
13460
13461	/**
13462	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
13463	*/
13464	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13465	{
13466	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13467	/** @todo Use VM-exit instruction information. */
13468	return VERR_EM_INTERPRETER;
13469	}
13470
13471
13472	/**
13473	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
13474	* Error VM-exit.
13475	*/
13476	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13477	{
13478	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13479	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13480	AssertRCReturn(rc, rc);
13481
13482	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo);
13483	if (RT_FAILURE(rc))
13484	return rc;
13485
13486	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
13487	NOREF(uInvalidReason);
13488
13489	#ifdef VBOX_STRICT
13490	uint32_t fIntrState;
13491	RTHCUINTREG uHCReg;
13492	uint64_t u64Val;
13493	uint32_t u32Val;
13494	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
13495	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
13496	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
13497	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13498	AssertRCReturn(rc, rc);
13499
13500	Log4(("uInvalidReason %u\n", uInvalidReason));
13501	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
13502	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
13503	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
13504	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
13505
13506	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
13507	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
13508	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
13509	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
13510	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
13511	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13512	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
13513	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
13514	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
13515	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13516	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
13517	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
13518
13519	hmR0DumpRegs(pVCpu);
13520	#endif
13521
13522	return VERR_VMX_INVALID_GUEST_STATE;
13523	}
13524
13525
13526	/**
13527	* VM-exit handler for VM-entry failure due to an MSR-load
13528	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
13529	*/
13530	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13531	{
13532	AssertMsgFailed(("Unexpected MSR-load exit\n"));
13533	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13534	}
13535
13536
13537	/**
13538	* VM-exit handler for VM-entry failure due to a machine-check event
13539	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
13540	*/
13541	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13542	{
13543	AssertMsgFailed(("Unexpected machine-check event exit\n"));
13544	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13545	}
13546
13547
13548	/**
13549	* VM-exit handler for all undefined reasons. Should never ever happen.. in
13550	* theory.
13551	*/
13552	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13553	{
13554	RT_NOREF2(pVCpu, pVmxTransient);
13555	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
13556	return VERR_VMX_UNDEFINED_EXIT_CODE;
13557	}
13558
13559
13560	/**
13561	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
13562	* (VMX_EXIT_GDTR_IDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
13563	* Conditional VM-exit.
13564	*/
13565	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13566	{
13567	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13568
13569	/* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
13570	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
13571	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13572	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
13573	return VERR_EM_INTERPRETER;
13574	AssertMsgFailed(("Unexpected XDTR access\n"));
13575	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13576	}
13577
13578
13579	/**
13580	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
13581	*/
13582	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13583	{
13584	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13585
13586	/* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
13587	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13588	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
13589	return VERR_EM_INTERPRETER;
13590	AssertMsgFailed(("Unexpected RDRAND exit\n"));
13591	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13592	}
13593
13594
13595	/**
13596	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
13597	*/
13598	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13599	{
13600	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13601
13602	/** @todo Optimize this: We currently drag in in the whole MSR state
13603	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13604	* MSRs required. That would require changes to IEM and possibly CPUM too.
13605	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13606	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13607	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13608	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13609	switch (idMsr)
13610	{
13611	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13612	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13613	}
13614
13615	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13616	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13617	AssertRCReturn(rc, rc);
13618
13619	Log4Func(("ecx=%#RX32\n", idMsr));
13620
13621	#ifdef VBOX_STRICT
13622	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
13623	{
13624	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
13625	&& idMsr != MSR_K6_EFER)
13626	{
13627	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
13628	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13629	}
13630	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13631	{
13632	Assert(pVmcsInfo->pvMsrBitmap);
13633	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13634	if (fMsrpm & VMXMSRPM_ALLOW_RD)
13635	{
13636	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
13637	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13638	}
13639	}
13640	}
13641	#endif
13642
13643	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
13644	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
13645	if (rcStrict == VINF_SUCCESS)
13646	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13647	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
13648	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13649	{
13650	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13651	rcStrict = VINF_SUCCESS;
13652	}
13653	else
13654	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13655
13656	return rcStrict;
13657	}
13658
13659
13660	/**
13661	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
13662	*/
13663	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13664	{
13665	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13666
13667	/** @todo Optimize this: We currently drag in in the whole MSR state
13668	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13669	* MSRs required. That would require changes to IEM and possibly CPUM too.
13670	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13671	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13672	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13673
13674	/*
13675	* The FS and GS base MSRs are not part of the above all-MSRs mask.
13676	* Although we don't need to fetch the base as it will be overwritten shortly, while
13677	* loading guest-state we would also load the entire segment register including limit
13678	* and attributes and thus we need to load them here.
13679	*/
13680	switch (idMsr)
13681	{
13682	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13683	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13684	}
13685
13686	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13687	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13688	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13689	AssertRCReturn(rc, rc);
13690
13691	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
13692
13693	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
13694	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
13695
13696	if (rcStrict == VINF_SUCCESS)
13697	{
13698	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13699
13700	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
13701	if ( idMsr == MSR_IA32_APICBASE
13702	\|\| ( idMsr >= MSR_IA32_X2APIC_START
13703	&& idMsr <= MSR_IA32_X2APIC_END))
13704	{
13705	/*
13706	* We've already saved the APIC related guest-state (TPR) in post-run phase.
13707	* When full APIC register virtualization is implemented we'll have to make
13708	* sure APIC state is saved from the VMCS before IEM changes it.
13709	*/
13710	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
13711	}
13712	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
13713	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13714	else if (idMsr == MSR_K6_EFER)
13715	{
13716	/*
13717	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
13718	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
13719	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
13720	*/
13721	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
13722	}
13723
13724	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
13725	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
13726	{
13727	switch (idMsr)
13728	{
13729	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
13730	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
13731	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
13732	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
13733	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
13734	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
13735	default:
13736	{
13737	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
13738	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
13739	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13740	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
13741	break;
13742	}
13743	}
13744	}
13745	#ifdef VBOX_STRICT
13746	else
13747	{
13748	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
13749	switch (idMsr)
13750	{
13751	case MSR_IA32_SYSENTER_CS:
13752	case MSR_IA32_SYSENTER_EIP:
13753	case MSR_IA32_SYSENTER_ESP:
13754	case MSR_K8_FS_BASE:
13755	case MSR_K8_GS_BASE:
13756	{
13757	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13758	Assert(fMsrpm == (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR));
13759
13760	uint32_t u32Proc;
13761	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Proc);
13762	AssertRC(rc);
13763	Assert(u32Proc == pVmcsInfo->u32ProcCtls);
13764	Assert(u32Proc & VMX_PROC_CTLS_USE_MSR_BITMAPS);
13765
13766	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32 fMsrpm=%#RX32\n", idMsr, fMsrpm));
13767	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13768	}
13769
13770	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
13771	default:
13772	{
13773	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
13774	{
13775	/* EFER MSR writes are always intercepted. */
13776	if (idMsr != MSR_K6_EFER)
13777	{
13778	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
13779	idMsr));
13780	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13781	}
13782	}
13783
13784	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13785	{
13786	Assert(pVmcsInfo->pvMsrBitmap);
13787	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13788	if (fMsrpm & VMXMSRPM_ALLOW_WR)
13789	{
13790	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
13791	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13792	}
13793	}
13794	break;
13795	}
13796	}
13797	}
13798	#endif /* VBOX_STRICT */
13799	}
13800	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13801	{
13802	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13803	rcStrict = VINF_SUCCESS;
13804	}
13805	else
13806	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13807
13808	return rcStrict;
13809	}
13810
13811
13812	/**
13813	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
13814	*/
13815	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13816	{
13817	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13818	/** @todo The guest has likely hit a contended spinlock. We might want to
13819	* poke a schedule different guest VCPU. */
13820	return VINF_EM_RAW_INTERRUPT;
13821	}
13822
13823
13824	/**
13825	* VM-exit handler for when the TPR value is lowered below the specified
13826	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
13827	*/
13828	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13829	{
13830	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13831	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
13832
13833	/*
13834	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
13835	* We'll re-evaluate pending interrupts and inject them before the next VM
13836	* entry so we can just continue execution here.
13837	*/
13838	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
13839	return VINF_SUCCESS;
13840	}
13841
13842
13843	/**
13844	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
13845	* VM-exit.
13846	*
13847	* @retval VINF_SUCCESS when guest execution can continue.
13848	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
13849	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
13850	* interpreter.
13851	*/
13852	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13853	{
13854	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13855	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
13856
13857	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13858	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13859	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13860	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13861	AssertRCReturn(rc, rc);
13862
13863	VBOXSTRICTRC rcStrict;
13864	PVM pVM = pVCpu->CTX_SUFF(pVM);
13865	RTGCUINTPTR const uExitQual = pVmxTransient->uExitQual;
13866	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
13867	switch (uAccessType)
13868	{
13869	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
13870	{
13871	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
13872	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
13873	VMX_EXIT_QUAL_CRX_GENREG(uExitQual));
13874	AssertMsg( rcStrict == VINF_SUCCESS
13875	\|\| rcStrict == VINF_IEM_RAISED_XCPT
13876	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13877
13878	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
13879	{
13880	case 0:
13881	{
13882	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13883	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13884	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
13885	Log4Func(("CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
13886
13887	/*
13888	* This is a kludge for handling switches back to real mode when we try to use
13889	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
13890	* deal with special selector values, so we have to return to ring-3 and run
13891	* there till the selector values are V86 mode compatible.
13892	*
13893	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
13894	* latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
13895	* at the end of this function.
13896	*/
13897	if ( rc == VINF_SUCCESS
13898	&& !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
13899	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
13900	&& (uOldCr0 & X86_CR0_PE)
13901	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
13902	{
13903	/** @todo check selectors rather than returning all the time. */
13904	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
13905	rcStrict = VINF_EM_RESCHEDULE_REM;
13906	}
13907	break;
13908	}
13909
13910	case 2:
13911	{
13912	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
13913	/* Nothing to do here, CR2 it's not part of the VMCS. */
13914	break;
13915	}
13916
13917	case 3:
13918	{
13919	Assert( !pVM->hm.s.fNestedPaging
13920	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
13921	\|\| pVCpu->hm.s.fUsingDebugLoop);
13922	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
13923	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13924	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
13925	Log4Func(("CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
13926	break;
13927	}
13928
13929	case 4:
13930	{
13931	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
13932	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13933	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
13934	Log4Func(("CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
13935	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
13936	break;
13937	}
13938
13939	case 8:
13940	{
13941	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
13942	Assert(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
13943	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13944	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
13945	break;
13946	}
13947	default:
13948	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual)));
13949	break;
13950	}
13951	break;
13952	}
13953
13954	case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
13955	{
13956	Assert( !pVM->hm.s.fNestedPaging
13957	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
13958	\|\| pVCpu->hm.s.fUsingDebugLoop
13959	\|\| VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 3);
13960	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
13961	Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 8
13962	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
13963
13964	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_GENREG(uExitQual),
13965	VMX_EXIT_QUAL_CRX_REGISTER(uExitQual));
13966	AssertMsg( rcStrict == VINF_SUCCESS
13967	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13968	#ifdef VBOX_WITH_STATISTICS
13969	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
13970	{
13971	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
13972	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
13973	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
13974	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
13975	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
13976	}
13977	#endif
13978	Log4Func(("CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
13979	VBOXSTRICTRC_VAL(rcStrict)));
13980	if (VMX_EXIT_QUAL_CRX_GENREG(uExitQual) == X86_GREG_xSP)
13981	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
13982	else
13983	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13984	break;
13985	}
13986
13987	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
13988	{
13989	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
13990	AssertMsg( rcStrict == VINF_SUCCESS
13991	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13992
13993	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13994	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
13995	Log4Func(("CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
13996	break;
13997	}
13998
13999	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
14000	{
14001	/* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
14002	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
14003	AssertRCReturn(rc, rc);
14004	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual),
14005	pVmxTransient->uGuestLinearAddr);
14006	AssertMsg( rcStrict == VINF_SUCCESS
14007	\|\| rcStrict == VINF_IEM_RAISED_XCPT
14008	, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14009
14010	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14011	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
14012	Log4Func(("LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
14013	break;
14014	}
14015
14016	default:
14017	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
14018	VERR_VMX_UNEXPECTED_EXCEPTION);
14019	}
14020
14021	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
14022	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
14023	if (rcStrict == VINF_IEM_RAISED_XCPT)
14024	{
14025	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14026	rcStrict = VINF_SUCCESS;
14027	}
14028
14029	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
14030	NOREF(pVM);
14031	return rcStrict;
14032	}
14033
14034
14035	/**
14036	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
14037	* VM-exit.
14038	*/
14039	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14040	{
14041	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14042	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
14043
14044	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14045	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14046	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14047	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14048	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
14049	\| CPUMCTX_EXTRN_EFER);
14050	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
14051	AssertRCReturn(rc, rc);
14052
14053	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
14054	uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
14055	uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQual);
14056	bool fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
14057	bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
14058	bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
14059	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
14060	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
14061
14062	/*
14063	* Update exit history to see if this exit can be optimized.
14064	*/
14065	VBOXSTRICTRC rcStrict;
14066	PCEMEXITREC pExitRec = NULL;
14067	if ( !fGstStepping
14068	&& !fDbgStepping)
14069	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14070	!fIOString
14071	? !fIOWrite
14072	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
14073	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
14074	: !fIOWrite
14075	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
14076	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
14077	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14078	if (!pExitRec)
14079	{
14080	/* I/O operation lookup arrays. */
14081	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
14082	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
14083	uint32_t const cbValue = s_aIOSizes[uIOWidth];
14084	uint32_t const cbInstr = pVmxTransient->cbInstr;
14085	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
14086	PVM pVM = pVCpu->CTX_SUFF(pVM);
14087	if (fIOString)
14088	{
14089	/*
14090	* INS/OUTS - I/O String instruction.
14091	*
14092	* Use instruction-information if available, otherwise fall back on
14093	* interpreting the instruction.
14094	*/
14095	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14096	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
14097	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
14098	if (fInsOutsInfo)
14099	{
14100	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14101	AssertRCReturn(rc2, rc2);
14102	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
14103	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
14104	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
14105	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
14106	if (fIOWrite)
14107	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
14108	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
14109	else
14110	{
14111	/*
14112	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
14113	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
14114	* See Intel Instruction spec. for "INS".
14115	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
14116	*/
14117	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
14118	}
14119	}
14120	else
14121	rcStrict = IEMExecOne(pVCpu);
14122
14123	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14124	fUpdateRipAlready = true;
14125	}
14126	else
14127	{
14128	/*
14129	* IN/OUT - I/O instruction.
14130	*/
14131	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14132	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
14133	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
14134	if (fIOWrite)
14135	{
14136	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
14137	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
14138	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14139	&& !pCtx->eflags.Bits.u1TF)
14140	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
14141	}
14142	else
14143	{
14144	uint32_t u32Result = 0;
14145	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
14146	if (IOM_SUCCESS(rcStrict))
14147	{
14148	/* Save result of I/O IN instr. in AL/AX/EAX. */
14149	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
14150	}
14151	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14152	&& !pCtx->eflags.Bits.u1TF)
14153	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
14154	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
14155	}
14156	}
14157
14158	if (IOM_SUCCESS(rcStrict))
14159	{
14160	if (!fUpdateRipAlready)
14161	{
14162	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
14163	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14164	}
14165
14166	/*
14167	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
14168	* while booting Fedora 17 64-bit guest.
14169	*
14170	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
14171	*/
14172	if (fIOString)
14173	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
14174
14175	/*
14176	* If any I/O breakpoints are armed, we need to check if one triggered
14177	* and take appropriate action.
14178	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
14179	*/
14180	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
14181	AssertRCReturn(rc, rc);
14182
14183	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
14184	* execution engines about whether hyper BPs and such are pending. */
14185	uint32_t const uDr7 = pCtx->dr[7];
14186	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
14187	&& X86_DR7_ANY_RW_IO(uDr7)
14188	&& (pCtx->cr4 & X86_CR4_DE))
14189	\|\| DBGFBpIsHwIoArmed(pVM)))
14190	{
14191	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
14192
14193	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
14194	VMMRZCallRing3Disable(pVCpu);
14195	HM_DISABLE_PREEMPT(pVCpu);
14196
14197	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
14198
14199	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
14200	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
14201	{
14202	/* Raise #DB. */
14203	if (fIsGuestDbgActive)
14204	ASMSetDR6(pCtx->dr[6]);
14205	if (pCtx->dr[7] != uDr7)
14206	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
14207
14208	hmR0VmxSetPendingXcptDB(pVCpu);
14209	}
14210	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
14211	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
14212	else if ( rcStrict2 != VINF_SUCCESS
14213	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
14214	rcStrict = rcStrict2;
14215	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
14216
14217	HM_RESTORE_PREEMPT();
14218	VMMRZCallRing3Enable(pVCpu);
14219	}
14220	}
14221
14222	#ifdef VBOX_STRICT
14223	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14224	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
14225	Assert(!fIOWrite);
14226	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14227	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
14228	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
14229	Assert(fIOWrite);
14230	else
14231	{
14232	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
14233	* statuses, that the VMM device and some others may return. See
14234	* IOM_SUCCESS() for guidance. */
14235	AssertMsg( RT_FAILURE(rcStrict)
14236	\|\| rcStrict == VINF_SUCCESS
14237	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
14238	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
14239	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
14240	\|\| rcStrict == VINF_EM_RAW_TO_R3
14241	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14242	# endif
14243	}
14244	#endif
14245	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
14246	}
14247	else
14248	{
14249	/*
14250	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14251	*/
14252	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14253	AssertRCReturn(rc2, rc2);
14254	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
14255	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
14256	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
14257	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14258	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
14259	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
14260
14261	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14262	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14263
14264	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14265	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14266	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14267	}
14268	return rcStrict;
14269	}
14270
14271
14272	/**
14273	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
14274	* VM-exit.
14275	*/
14276	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14277	{
14278	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14279
14280	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
14281	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14282	AssertRCReturn(rc, rc);
14283	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
14284	{
14285	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14286	AssertRCReturn(rc, rc);
14287	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
14288	{
14289	uint32_t uErrCode;
14290	RTGCUINTPTR GCPtrFaultAddress;
14291	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
14292	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
14293	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
14294	if (fErrorCodeValid)
14295	{
14296	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14297	AssertRCReturn(rc, rc);
14298	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
14299	}
14300	else
14301	uErrCode = 0;
14302
14303	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
14304	&& uVector == X86_XCPT_PF)
14305	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
14306	else
14307	GCPtrFaultAddress = 0;
14308
14309	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14310	AssertRCReturn(rc, rc);
14311
14312	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
14313	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
14314
14315	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
14316	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14317	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14318	}
14319	}
14320
14321	/* Fall back to the interpreter to emulate the task-switch. */
14322	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14323	return VERR_EM_INTERPRETER;
14324	}
14325
14326
14327	/**
14328	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
14329	*/
14330	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14331	{
14332	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14333
14334	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14335	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
14336	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14337	AssertRCReturn(rc, rc);
14338	return VINF_EM_DBG_STEPPED;
14339	}
14340
14341
14342	/**
14343	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
14344	*/
14345	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14346	{
14347	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14348	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
14349
14350	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14351	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14352	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14353	{
14354	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
14355	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14356	{
14357	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14358	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14359	}
14360	}
14361	else
14362	{
14363	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14364	rcStrict1 = VINF_SUCCESS;
14365	return rcStrict1;
14366	}
14367
14368	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
14369	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14370	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14371	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14372	AssertRCReturn(rc, rc);
14373
14374	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
14375	uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
14376	VBOXSTRICTRC rcStrict2;
14377	switch (uAccessType)
14378	{
14379	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
14380	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
14381	{
14382	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
14383	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
14384	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
14385
14386	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
14387	GCPhys &= PAGE_BASE_GC_MASK;
14388	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
14389	PVM pVM = pVCpu->CTX_SUFF(pVM);
14390	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
14391	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
14392
14393	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14394	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
14395	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
14396	CPUMCTX2CORE(pCtx), GCPhys);
14397	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14398	if ( rcStrict2 == VINF_SUCCESS
14399	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14400	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14401	{
14402	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14403	\| HM_CHANGED_GUEST_APIC_TPR);
14404	rcStrict2 = VINF_SUCCESS;
14405	}
14406	break;
14407	}
14408
14409	default:
14410	Log4Func(("uAccessType=%#x\n", uAccessType));
14411	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
14412	break;
14413	}
14414
14415	if (rcStrict2 != VINF_SUCCESS)
14416	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
14417	return rcStrict2;
14418	}
14419
14420
14421	/**
14422	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
14423	* VM-exit.
14424	*/
14425	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14426	{
14427	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14428
14429	/* We should -not- get this VM-exit if the guest's debug registers were active. */
14430	if (pVmxTransient->fWasGuestDebugStateActive)
14431	{
14432	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
14433	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14434	}
14435
14436	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14437	if ( !pVCpu->hm.s.fSingleInstruction
14438	&& !pVmxTransient->fWasHyperDebugStateActive)
14439	{
14440	Assert(!DBGFIsStepping(pVCpu));
14441	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
14442
14443	/* Don't intercept MOV DRx any more. */
14444	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
14445	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14446	AssertRCReturn(rc, rc);
14447
14448	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
14449	VMMRZCallRing3Disable(pVCpu);
14450	HM_DISABLE_PREEMPT(pVCpu);
14451
14452	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
14453	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
14454	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
14455
14456	HM_RESTORE_PREEMPT();
14457	VMMRZCallRing3Enable(pVCpu);
14458
14459	#ifdef VBOX_WITH_STATISTICS
14460	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14461	AssertRCReturn(rc, rc);
14462	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14463	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14464	else
14465	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14466	#endif
14467	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
14468	return VINF_SUCCESS;
14469	}
14470
14471	/*
14472	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
14473	* The EFER MSR is always up-to-date.
14474	* Update the segment registers and DR7 from the CPU.
14475	*/
14476	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14477	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14478	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
14479	AssertRCReturn(rc, rc);
14480	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
14481
14482	PVM pVM = pVCpu->CTX_SUFF(pVM);
14483	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14484	{
14485	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14486	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
14487	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
14488	if (RT_SUCCESS(rc))
14489	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
14490	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14491	}
14492	else
14493	{
14494	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14495	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
14496	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
14497	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14498	}
14499
14500	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
14501	if (RT_SUCCESS(rc))
14502	{
14503	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14504	AssertRCReturn(rc2, rc2);
14505	return VINF_SUCCESS;
14506	}
14507	return rc;
14508	}
14509
14510
14511	/**
14512	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
14513	* Conditional VM-exit.
14514	*/
14515	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14516	{
14517	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14518	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14519
14520	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14521	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14522	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14523	{
14524	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
14525	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
14526	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14527	{
14528	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14529	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14530	}
14531	}
14532	else
14533	{
14534	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14535	rcStrict1 = VINF_SUCCESS;
14536	return rcStrict1;
14537	}
14538
14539	/*
14540	* Get sufficent state and update the exit history entry.
14541	*/
14542	RTGCPHYS GCPhys;
14543	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14544	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14545	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14546	AssertRCReturn(rc, rc);
14547
14548	VBOXSTRICTRC rcStrict;
14549	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14550	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
14551	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14552	if (!pExitRec)
14553	{
14554	/*
14555	* If we succeed, resume guest execution.
14556	* If we fail in interpreting the instruction because we couldn't get the guest physical address
14557	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
14558	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
14559	* weird case. See @bugref{6043}.
14560	*/
14561	PVM pVM = pVCpu->CTX_SUFF(pVM);
14562	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14563	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
14564	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
14565	if ( rcStrict == VINF_SUCCESS
14566	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
14567	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
14568	{
14569	/* Successfully handled MMIO operation. */
14570	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14571	\| HM_CHANGED_GUEST_APIC_TPR);
14572	rcStrict = VINF_SUCCESS;
14573	}
14574	}
14575	else
14576	{
14577	/*
14578	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14579	*/
14580	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14581	AssertRCReturn(rc2, rc2);
14582
14583	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
14584	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
14585
14586	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14587	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14588
14589	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14590	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14591	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14592	}
14593	return VBOXSTRICTRC_TODO(rcStrict);
14594	}
14595
14596
14597	/**
14598	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
14599	* VM-exit.
14600	*/
14601	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14602	{
14603	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14604	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14605
14606	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14607	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14608	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14609	{
14610	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
14611	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14612	Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
14613	}
14614	else
14615	{
14616	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14617	rcStrict1 = VINF_SUCCESS;
14618	return rcStrict1;
14619	}
14620
14621	RTGCPHYS GCPhys;
14622	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14623	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14624	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14625	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14626	AssertRCReturn(rc, rc);
14627
14628	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
14629	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
14630
14631	RTGCUINT uErrorCode = 0;
14632	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
14633	uErrorCode \|= X86_TRAP_PF_ID;
14634	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
14635	uErrorCode \|= X86_TRAP_PF_RW;
14636	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
14637	uErrorCode \|= X86_TRAP_PF_P;
14638
14639	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
14640
14641
14642	/* Handle the pagefault trap for the nested shadow table. */
14643	PVM pVM = pVCpu->CTX_SUFF(pVM);
14644	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14645
14646	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
14647	pCtx->cs.Sel, pCtx->rip));
14648
14649	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
14650	TRPMResetTrap(pVCpu);
14651
14652	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
14653	if ( rcStrict2 == VINF_SUCCESS
14654	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14655	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14656	{
14657	/* Successfully synced our nested page tables. */
14658	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
14659	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
14660	return VINF_SUCCESS;
14661	}
14662
14663	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14664	return rcStrict2;
14665	}
14666
14667	/** @} */
14668
14669	/** @name VM-exit exception handlers.
14670	* @{
14671	*/
14672	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14673	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14674	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14675
14676	/**
14677	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
14678	*/
14679	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14680	{
14681	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14682	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
14683
14684	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
14685	AssertRCReturn(rc, rc);
14686
14687	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
14688	{
14689	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
14690	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
14691
14692	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
14693	* provides VM-exit instruction length. If this causes problem later,
14694	* disassemble the instruction like it's done on AMD-V. */
14695	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14696	AssertRCReturn(rc2, rc2);
14697	return rc;
14698	}
14699
14700	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14701	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14702	return rc;
14703	}
14704
14705
14706	/**
14707	* VM-exit exception handler for \#BP (Breakpoint exception).
14708	*/
14709	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14710	{
14711	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14712	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
14713
14714	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14715	AssertRCReturn(rc, rc);
14716
14717	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14718	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14719	if (rc == VINF_EM_RAW_GUEST_TRAP)
14720	{
14721	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14722	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14723	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14724	AssertRCReturn(rc, rc);
14725
14726	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14727	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14728	}
14729
14730	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
14731	return rc;
14732	}
14733
14734
14735	/**
14736	* VM-exit exception handler for \#AC (alignment check exception).
14737	*/
14738	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14739	{
14740	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14741
14742	/*
14743	* Re-inject it. We'll detect any nesting before getting here.
14744	*/
14745	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14746	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14747	AssertRCReturn(rc, rc);
14748	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
14749
14750	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14751	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14752	return VINF_SUCCESS;
14753	}
14754
14755
14756	/**
14757	* VM-exit exception handler for \#DB (Debug exception).
14758	*/
14759	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14760	{
14761	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14762	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
14763
14764	/*
14765	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
14766	* for processing.
14767	*/
14768	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14769
14770	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
14771	uint64_t uDR6 = X86_DR6_INIT_VAL;
14772	uDR6 \|= (pVmxTransient->uExitQual & (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
14773
14774	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14775	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
14776	Log6Func(("rc=%Rrc\n", rc));
14777	if (rc == VINF_EM_RAW_GUEST_TRAP)
14778	{
14779	/*
14780	* The exception was for the guest. Update DR6, DR7.GD and
14781	* IA32_DEBUGCTL.LBR before forwarding it.
14782	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
14783	*/
14784	VMMRZCallRing3Disable(pVCpu);
14785	HM_DISABLE_PREEMPT(pVCpu);
14786
14787	pCtx->dr[6] &= ~X86_DR6_B_MASK;
14788	pCtx->dr[6] \|= uDR6;
14789	if (CPUMIsGuestDebugStateActive(pVCpu))
14790	ASMSetDR6(pCtx->dr[6]);
14791
14792	HM_RESTORE_PREEMPT();
14793	VMMRZCallRing3Enable(pVCpu);
14794
14795	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
14796	AssertRCReturn(rc, rc);
14797
14798	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
14799	pCtx->dr[7] &= ~X86_DR7_GD;
14800
14801	/* Paranoia. */
14802	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
14803	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
14804
14805	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
14806	AssertRCReturn(rc, rc);
14807
14808	/*
14809	* Raise #DB in the guest.
14810	*
14811	* It is important to reflect exactly what the VM-exit gave us (preserving the
14812	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
14813	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
14814	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
14815	*
14816	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
14817	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
14818	*/
14819	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14820	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14821	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14822	AssertRCReturn(rc, rc);
14823	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14824	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14825	return VINF_SUCCESS;
14826	}
14827
14828	/*
14829	* Not a guest trap, must be a hypervisor related debug event then.
14830	* Update DR6 in case someone is interested in it.
14831	*/
14832	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
14833	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
14834	CPUMSetHyperDR6(pVCpu, uDR6);
14835
14836	return rc;
14837	}
14838
14839
14840	/**
14841	* Hacks its way around the lovely mesa driver's backdoor accesses.
14842	*
14843	* @sa hmR0SvmHandleMesaDrvGp
14844	*/
14845	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14846	{
14847	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
14848	RT_NOREF(pCtx);
14849
14850	/* For now we'll just skip the instruction. */
14851	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14852	}
14853
14854
14855	/**
14856	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
14857	* backdoor logging w/o checking what it is running inside.
14858	*
14859	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
14860	* backdoor port and magic numbers loaded in registers.
14861	*
14862	* @returns true if it is, false if it isn't.
14863	* @sa hmR0SvmIsMesaDrvGp
14864	*/
14865	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14866	{
14867	/* 0xed: IN eAX,dx */
14868	uint8_t abInstr[1];
14869	if (pVmxTransient->cbInstr != sizeof(abInstr))
14870	return false;
14871
14872	/* Check that it is #GP(0). */
14873	if (pVmxTransient->uExitIntErrorCode != 0)
14874	return false;
14875
14876	/* Check magic and port. */
14877	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
14878	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
14879	if (pCtx->rax != UINT32_C(0x564d5868))
14880	return false;
14881	if (pCtx->dx != UINT32_C(0x5658))
14882	return false;
14883
14884	/* Flat ring-3 CS. */
14885	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
14886	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
14887	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
14888	if (pCtx->cs.Attr.n.u2Dpl != 3)
14889	return false;
14890	if (pCtx->cs.u64Base != 0)
14891	return false;
14892
14893	/* Check opcode. */
14894	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
14895	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
14896	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
14897	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
14898	if (RT_FAILURE(rc))
14899	return false;
14900	if (abInstr[0] != 0xed)
14901	return false;
14902
14903	return true;
14904	}
14905
14906
14907	/**
14908	* VM-exit exception handler for \#GP (General-protection exception).
14909	*
14910	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
14911	*/
14912	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14913	{
14914	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14915	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
14916
14917	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14918	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14919	if (pVmcsInfo->RealMode.fRealOnV86Active)
14920	{ /* likely */ }
14921	else
14922	{
14923	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14924	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv);
14925	#endif
14926	/* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
14927	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14928	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14929	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14930	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14931	AssertRCReturn(rc, rc);
14932	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
14933	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
14934
14935	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
14936	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
14937	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14938	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14939	else
14940	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
14941	return rc;
14942	}
14943
14944	Assert(CPUMIsGuestInRealModeEx(pCtx));
14945	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
14946
14947	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14948	AssertRCReturn(rc, rc);
14949
14950	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
14951	if (rcStrict == VINF_SUCCESS)
14952	{
14953	if (!CPUMIsGuestInRealModeEx(pCtx))
14954	{
14955	/*
14956	* The guest is no longer in real-mode, check if we can continue executing the
14957	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
14958	*/
14959	pVmcsInfo->RealMode.fRealOnV86Active = false;
14960	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
14961	{
14962	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
14963	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14964	}
14965	else
14966	{
14967	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
14968	rcStrict = VINF_EM_RESCHEDULE;
14969	}
14970	}
14971	else
14972	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14973	}
14974	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14975	{
14976	rcStrict = VINF_SUCCESS;
14977	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14978	}
14979	return VBOXSTRICTRC_VAL(rcStrict);
14980	}
14981
14982
14983	/**
14984	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
14985	* the exception reported in the VMX transient structure back into the VM.
14986	*
14987	* @remarks Requires uExitIntInfo in the VMX transient structure to be
14988	* up-to-date.
14989	*/
14990	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14991	{
14992	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14993	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14994	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14995	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active,
14996	("uVector=%#x u32XcptBitmap=%#X32\n",
14997	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
14998	NOREF(pVmcsInfo);
14999	#endif
15000
15001	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
15002	hmR0VmxCheckExitDueToEventDelivery(). */
15003	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15004	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15005	AssertRCReturn(rc, rc);
15006	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15007
15008	#ifdef DEBUG_ramshankar
15009	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
15010	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n",
15011	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pCtx->cs.Sel, pCtx->rip));
15012	#endif
15013
15014	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15015	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15016	return VINF_SUCCESS;
15017	}
15018
15019
15020	/**
15021	* VM-exit exception handler for \#PF (Page-fault exception).
15022	*/
15023	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15024	{
15025	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15026	PVM pVM = pVCpu->CTX_SUFF(pVM);
15027	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15028	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15029	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15030	AssertRCReturn(rc, rc);
15031
15032	if (!pVM->hm.s.fNestedPaging)
15033	{ /* likely */ }
15034	else
15035	{
15036	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
15037	Assert(pVCpu->hm.s.fUsingDebugLoop);
15038	#endif
15039	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
15040	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
15041	{
15042	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15043	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
15044	}
15045	else
15046	{
15047	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15048	hmR0VmxSetPendingXcptDF(pVCpu);
15049	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
15050	}
15051	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15052	return rc;
15053	}
15054
15055	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
15056	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
15057	if (pVmxTransient->fVectoringPF)
15058	{
15059	Assert(pVCpu->hm.s.Event.fPending);
15060	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15061	}
15062
15063	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15064	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15065	AssertRCReturn(rc, rc);
15066
15067	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
15068	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
15069
15070	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
15071	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
15072
15073	Log4Func(("#PF: rc=%Rrc\n", rc));
15074	if (rc == VINF_SUCCESS)
15075	{
15076	/*
15077	* This is typically a shadow page table sync or a MMIO instruction. But we may have
15078	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
15079	*/
15080	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15081	TRPMResetTrap(pVCpu);
15082	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
15083	return rc;
15084	}
15085
15086	if (rc == VINF_EM_RAW_GUEST_TRAP)
15087	{
15088	if (!pVmxTransient->fVectoringDoublePF)
15089	{
15090	/* It's a guest page fault and needs to be reflected to the guest. */
15091	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
15092	TRPMResetTrap(pVCpu);
15093	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
15094	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15095	uGstErrorCode, pVmxTransient->uExitQual);
15096	}
15097	else
15098	{
15099	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15100	TRPMResetTrap(pVCpu);
15101	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
15102	hmR0VmxSetPendingXcptDF(pVCpu);
15103	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
15104	}
15105
15106	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15107	return VINF_SUCCESS;
15108	}
15109
15110	TRPMResetTrap(pVCpu);
15111	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
15112	return rc;
15113	}
15114
15115	/** @} */
15116
15117	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15118	/** @name VMX instruction handlers.
15119	* @{
15120	*/
15121	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15122	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VMX instructions VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15123	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15124
15125	/**
15126	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
15127	*/
15128	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15129	{
15130	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15131
15132	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15133	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15134	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15135	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15136	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15137	AssertRCReturn(rc, rc);
15138
15139	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15140
15141	VMXVEXITINFO ExitInfo;
15142	RT_ZERO(ExitInfo);
15143	ExitInfo.uReason = pVmxTransient->uExitReason;
15144	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15145	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15146	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15147	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15148
15149	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
15150	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15151	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15152	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15153	{
15154	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15155	rcStrict = VINF_SUCCESS;
15156	}
15157	return rcStrict;
15158	}
15159
15160
15161	/**
15162	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
15163	*/
15164	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15165	{
15166	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15167
15168	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
15169	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15170	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15171	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15172	AssertRCReturn(rc, rc);
15173
15174	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15175
15176	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
15177	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15178	{
15179	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15180	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15181	}
15182	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15183	return rcStrict;
15184	}
15185
15186
15187	/**
15188	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
15189	*/
15190	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15191	{
15192	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15193
15194	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15195	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15196	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15197	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15198	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15199	AssertRCReturn(rc, rc);
15200
15201	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15202
15203	VMXVEXITINFO ExitInfo;
15204	RT_ZERO(ExitInfo);
15205	ExitInfo.uReason = pVmxTransient->uExitReason;
15206	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15207	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15208	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15209	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15210
15211	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
15212	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15213	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15214	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15215	{
15216	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15217	rcStrict = VINF_SUCCESS;
15218	}
15219	return rcStrict;
15220	}
15221
15222
15223	/**
15224	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
15225	*/
15226	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15227	{
15228	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15229
15230	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15231	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15232	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15233	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15234	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15235	AssertRCReturn(rc, rc);
15236
15237	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15238
15239	VMXVEXITINFO ExitInfo;
15240	RT_ZERO(ExitInfo);
15241	ExitInfo.uReason = pVmxTransient->uExitReason;
15242	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15243	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15244	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15245	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15246
15247	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
15248	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15249	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15250	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15251	{
15252	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15253	rcStrict = VINF_SUCCESS;
15254	}
15255	return rcStrict;
15256	}
15257
15258
15259	/**
15260	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Unconditional VM-exit.
15261	*/
15262	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15263	{
15264	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15265
15266	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15267	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15268	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15269	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15270	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15271	AssertRCReturn(rc, rc);
15272
15273	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15274
15275	VMXVEXITINFO ExitInfo;
15276	RT_ZERO(ExitInfo);
15277	ExitInfo.uReason = pVmxTransient->uExitReason;
15278	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15279	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15280	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15281	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15282	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15283
15284	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
15285	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15286	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15287	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15288	{
15289	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15290	rcStrict = VINF_SUCCESS;
15291	}
15292	return rcStrict;
15293	}
15294
15295
15296	/**
15297	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
15298	*/
15299	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15300	{
15301	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15302
15303	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
15304	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15305	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15306	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15307	AssertRCReturn(rc, rc);
15308
15309	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15310
15311	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
15312	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15313	{
15314	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15315	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15316	}
15317	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15318	return rcStrict;
15319	}
15320
15321
15322	/**
15323	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Unconditional VM-exit.
15324	*/
15325	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15326	{
15327	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15328
15329	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15330	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15331	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15332	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15333	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15334	AssertRCReturn(rc, rc);
15335
15336	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15337
15338	VMXVEXITINFO ExitInfo;
15339	RT_ZERO(ExitInfo);
15340	ExitInfo.uReason = pVmxTransient->uExitReason;
15341	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15342	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15343	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15344	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15345	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15346
15347	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
15348	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15349	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15350	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15351	{
15352	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15353	rcStrict = VINF_SUCCESS;
15354	}
15355	return rcStrict;
15356	}
15357
15358
15359	/**
15360	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
15361	*/
15362	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15363	{
15364	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15365
15366	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15367	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
15368	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15369	AssertRCReturn(rc, rc);
15370
15371	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15372
15373	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
15374	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15375	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
15376	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15377	{
15378	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15379	rcStrict = VINF_SUCCESS;
15380	}
15381	return rcStrict;
15382	}
15383
15384
15385	/**
15386	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
15387	*/
15388	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15389	{
15390	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15391
15392	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15393	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15394	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15395	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15396	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15397	AssertRCReturn(rc, rc);
15398
15399	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15400
15401	VMXVEXITINFO ExitInfo;
15402	RT_ZERO(ExitInfo);
15403	ExitInfo.uReason = pVmxTransient->uExitReason;
15404	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15405	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15406	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15407	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15408
15409	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
15410	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15411	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15412	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15413	{
15414	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15415	rcStrict = VINF_SUCCESS;
15416	}
15417	return rcStrict;
15418	}
15419
15420	/** @} */
15421	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
15422

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 78223

Download in other formats: