HMVMXR0.cpp@ 78259

Last change on this file since 78259 was 78259, checked in by vboxsync, 6 years ago
VMM/HMVMXR0: Nested VMX: bugref:9180 Use the VMCS info. from the VMX-transient object. It should be valid at this point.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 641.4 KB

Line
1	/* $Id: HMVMXR0.cpp 78259 2019-04-23 10:40:45Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27
28	#include <VBox/vmm/pdmapi.h>
29	#include <VBox/vmm/dbgf.h>
30	#include <VBox/vmm/iem.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#include <VBox/vmm/em.h>
35	#include <VBox/vmm/gim.h>
36	#include <VBox/vmm/apic.h>
37	#ifdef VBOX_WITH_REM
38	# include <VBox/vmm/rem.h>
39	#endif
40	#include "HMInternal.h"
41	#include <VBox/vmm/vm.h>
42	#include <VBox/vmm/hmvmxinline.h>
43	#include "HMVMXR0.h"
44	#include "dtrace/VBoxVMM.h"
45
46	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
47	#ifdef DEBUG_ramshankar
48	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
49	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
50	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
51	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
52	# define HMVMX_ALWAYS_TRAP_PF
53	# define HMVMX_ALWAYS_FLUSH_TLB
54	# define HMVMX_ALWAYS_SWAP_EFER
55	#endif
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	/** Use the function table. */
62	#define HMVMX_USE_FUNCTION_TABLE
63
64	/** Determine which tagged-TLB flush handler to use. */
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70	/** @name HMVMX_READ_XXX
71	* Flags to skip redundant reads of some common VMCS fields that are not part of
72	* the guest-CPU or VCPU state but are needed while handling VM-exits.
73	*/
74	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
75	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
76	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
77	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
78	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
79	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
80	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
81	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
82	/** @} */
83
84	/**
85	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
86	* guest using hardware-assisted VMX.
87	*
88	* This excludes state like GPRs (other than RSP) which are always are
89	* swapped and restored across the world-switch and also registers like EFER,
90	* MSR which cannot be modified by the guest without causing a VM-exit.
91	*/
92	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
93	\| CPUMCTX_EXTRN_RFLAGS \
94	\| CPUMCTX_EXTRN_RSP \
95	\| CPUMCTX_EXTRN_SREG_MASK \
96	\| CPUMCTX_EXTRN_TABLE_MASK \
97	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
98	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
99	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
100	\| CPUMCTX_EXTRN_TSC_AUX \
101	\| CPUMCTX_EXTRN_OTHER_MSRS \
102	\| CPUMCTX_EXTRN_CR0 \
103	\| CPUMCTX_EXTRN_CR3 \
104	\| CPUMCTX_EXTRN_CR4 \
105	\| CPUMCTX_EXTRN_DR7 \
106	\| CPUMCTX_EXTRN_HM_VMX_MASK)
107
108	/**
109	* Exception bitmap mask for real-mode guests (real-on-v86).
110	*
111	* We need to intercept all exceptions manually except:
112	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
113	* due to bugs in Intel CPUs.
114	* - \#PF need not be intercepted even in real-mode if we have nested paging
115	* support.
116	*/
117	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
118	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
119	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
120	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
121	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
122	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
123	\| RT_BIT(X86_XCPT_XF))
124
125	/** Maximum VM-instruction error number. */
126	#define HMVMX_INSTR_ERROR_MAX 28
127
128	/** Profiling macro. */
129	#ifdef HM_PROFILE_EXIT_DISPATCH
130	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
131	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
132	#else
133	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
134	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
135	#endif
136
137	/** Assert that preemption is disabled or covered by thread-context hooks. */
138	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
139	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
140
141	/** Assert that we haven't migrated CPUs when thread-context hooks are not
142	* used. */
143	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
144	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
145	("Illegal migration! Entered on CPU %u Current %u\n", \
146	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
147
148	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
149	* context. */
150	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
151	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
152	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
153
154	/** Helper macro for VM-exit handlers called unexpectedly. */
155	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
156	do { \
157	(a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
158	return VERR_VMX_UNEXPECTED_EXIT; \
159	} while (0)
160
161	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
162	/** Macro that does the necessary privilege checks and intercepted VM-exits for
163	* guests that attempted to execute a VMX instruction. */
164	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
165	do \
166	{ \
167	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
168	if (rcStrictTmp == VINF_SUCCESS) \
169	{ /* likely */ } \
170	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
171	{ \
172	Assert((a_pVCpu)->hm.s.Event.fPending); \
173	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
174	return VINF_SUCCESS; \
175	} \
176	else \
177	{ \
178	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
179	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
180	} \
181	} while (0)
182
183	/** Macro that decodes a memory operand for an instruction VM-exit. */
184	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
185	do \
186	{ \
187	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
188	(a_pGCPtrEffAddr)); \
189	if (rcStrictTmp == VINF_SUCCESS) \
190	{ /* likely */ } \
191	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
192	{ \
193	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
194	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
195	NOREF(uXcptTmp); \
196	return VINF_SUCCESS; \
197	} \
198	else \
199	{ \
200	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
201	return rcStrictTmp; \
202	} \
203	} while (0)
204
205	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
206
207
208	/*********************************************************************************************************************************
209	* Structures and Typedefs *
210	*********************************************************************************************************************************/
211	/**
212	* VMX transient state.
213	*
214	* A state structure for holding miscellaneous information across
215	* VMX non-root operation and restored after the transition.
216	*/
217	typedef struct VMXTRANSIENT
218	{
219	/** The host's rflags/eflags. */
220	RTCCUINTREG fEFlags;
221	#if HC_ARCH_BITS == 32
222	uint32_t u32Alignment0;
223	#endif
224	/** The guest's TPR value used for TPR shadowing. */
225	uint8_t u8GuestTpr;
226	/** Alignment. */
227	uint8_t abAlignment0[7];
228
229	/** The basic VM-exit reason. */
230	uint16_t uExitReason;
231	/** Alignment. */
232	uint16_t u16Alignment0;
233	/** The VM-exit interruption error code. */
234	uint32_t uExitIntErrorCode;
235	/** The VM-exit exit code qualification. */
236	uint64_t uExitQual;
237	/** The Guest-linear address. */
238	uint64_t uGuestLinearAddr;
239
240	/** The VM-exit interruption-information field. */
241	uint32_t uExitIntInfo;
242	/** The VM-exit instruction-length field. */
243	uint32_t cbInstr;
244	/** The VM-exit instruction-information field. */
245	VMXEXITINSTRINFO ExitInstrInfo;
246	/** Whether the VM-entry failed or not. */
247	bool fVMEntryFailed;
248	/** Whether we are currently executing a nested-guest. */
249	bool fIsNestedGuest;
250	/** Alignment. */
251	uint8_t abAlignment1[2];
252
253	/** The VM-entry interruption-information field. */
254	uint32_t uEntryIntInfo;
255	/** The VM-entry exception error code field. */
256	uint32_t uEntryXcptErrorCode;
257	/** The VM-entry instruction length field. */
258	uint32_t cbEntryInstr;
259
260	/** IDT-vectoring information field. */
261	uint32_t uIdtVectoringInfo;
262	/** IDT-vectoring error code. */
263	uint32_t uIdtVectoringErrorCode;
264
265	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
266	uint32_t fVmcsFieldsRead;
267
268	/** Whether the guest debug state was active at the time of VM-exit. */
269	bool fWasGuestDebugStateActive;
270	/** Whether the hyper debug state was active at the time of VM-exit. */
271	bool fWasHyperDebugStateActive;
272	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
273	bool fUpdatedTscOffsettingAndPreemptTimer;
274	/** Whether the VM-exit was caused by a page-fault during delivery of a
275	* contributory exception or a page-fault. */
276	bool fVectoringDoublePF;
277	/** Whether the VM-exit was caused by a page-fault during delivery of an
278	* external interrupt or NMI. */
279	bool fVectoringPF;
280	bool afAlignment0[3];
281
282	/** The VMCS info. object. */
283	PVMXVMCSINFO pVmcsInfo;
284	} VMXTRANSIENT;
285	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
286	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
287	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
288	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
289	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
290	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
291	/** Pointer to VMX transient state. */
292	typedef VMXTRANSIENT *PVMXTRANSIENT;
293
294	/**
295	* Memory operand read or write access.
296	*/
297	typedef enum VMXMEMACCESS
298	{
299	VMXMEMACCESS_READ = 0,
300	VMXMEMACCESS_WRITE = 1
301	} VMXMEMACCESS;
302
303	/**
304	* VMX VM-exit handler.
305	*
306	* @returns Strict VBox status code (i.e. informational status codes too).
307	* @param pVCpu The cross context virtual CPU structure.
308	* @param pVmxTransient The VMX-transient structure.
309	*/
310	#ifndef HMVMX_USE_FUNCTION_TABLE
311	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
312	#else
313	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
314	/** Pointer to VM-exit handler. */
315	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
316	#endif
317
318	/**
319	* VMX VM-exit handler, non-strict status code.
320	*
321	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
322	*
323	* @returns VBox status code, no informational status code returned.
324	* @param pVCpu The cross context virtual CPU structure.
325	* @param pVmxTransient The VMX-transient structure.
326	*
327	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
328	* use of that status code will be replaced with VINF_EM_SOMETHING
329	* later when switching over to IEM.
330	*/
331	#ifndef HMVMX_USE_FUNCTION_TABLE
332	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
333	#else
334	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
335	#endif
336
337
338	/*********************************************************************************************************************************
339	* Internal Functions *
340	*********************************************************************************************************************************/
341	#ifndef HMVMX_USE_FUNCTION_TABLE
342	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
343	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
344	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
345	#else
346	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
347	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
348	#endif
349	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
350	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
351	#endif
352
353	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
354	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
355	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
356	#endif
357
358	/** @name VM-exit handlers.
359	* @{
360	*/
361	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
362	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
363	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
364	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
365	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
366	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
367	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
368	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
369	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
370	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
371	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
372	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
373	static FNVMXEXITHANDLER hmR0VmxExitHlt;
374	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
375	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
376	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
377	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
378	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
379	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
380	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
381	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
382	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
383	static FNVMXEXITHANDLER hmR0VmxExitVmread;
384	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
385	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
386	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
387	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
388	#endif
389	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
390	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
391	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
392	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
393	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
394	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
395	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
396	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
397	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
398	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
399	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
400	static FNVMXEXITHANDLER hmR0VmxExitMwait;
401	static FNVMXEXITHANDLER hmR0VmxExitMtf;
402	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
403	static FNVMXEXITHANDLER hmR0VmxExitPause;
404	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
405	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
406	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
407	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
408	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
409	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
410	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
411	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
412	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
413	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
414	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
415	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
416	/** @} */
417
418	/** @name Helpers for hardware exceptions VM-exit handlers.
419	* @{
420	*/
421	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
422	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
423	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
424	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
425	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
426	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
427	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
428	/** @} */
429
430
431	/*********************************************************************************************************************************
432	* Global Variables *
433	*********************************************************************************************************************************/
434	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
435	static const uint32_t g_aVmcsCacheSegBase[] =
436	{
437	VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
438	VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
439	VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
440	VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
441	VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
442	VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
443	};
444	AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
445	#endif
446	static const uint32_t g_aVmcsSegBase[] =
447	{
448	VMX_VMCS_GUEST_ES_BASE,
449	VMX_VMCS_GUEST_CS_BASE,
450	VMX_VMCS_GUEST_SS_BASE,
451	VMX_VMCS_GUEST_DS_BASE,
452	VMX_VMCS_GUEST_FS_BASE,
453	VMX_VMCS_GUEST_GS_BASE
454	};
455	static const uint32_t g_aVmcsSegSel[] =
456	{
457	VMX_VMCS16_GUEST_ES_SEL,
458	VMX_VMCS16_GUEST_CS_SEL,
459	VMX_VMCS16_GUEST_SS_SEL,
460	VMX_VMCS16_GUEST_DS_SEL,
461	VMX_VMCS16_GUEST_FS_SEL,
462	VMX_VMCS16_GUEST_GS_SEL
463	};
464	static const uint32_t g_aVmcsSegLimit[] =
465	{
466	VMX_VMCS32_GUEST_ES_LIMIT,
467	VMX_VMCS32_GUEST_CS_LIMIT,
468	VMX_VMCS32_GUEST_SS_LIMIT,
469	VMX_VMCS32_GUEST_DS_LIMIT,
470	VMX_VMCS32_GUEST_FS_LIMIT,
471	VMX_VMCS32_GUEST_GS_LIMIT
472	};
473	static const uint32_t g_aVmcsSegAttr[] =
474	{
475	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
476	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
477	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
478	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
479	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
480	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
481	};
482	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
483	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
484	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
485	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
486
487	#ifdef HMVMX_USE_FUNCTION_TABLE
488	/**
489	* VMX_EXIT dispatch table.
490	*/
491	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
492	{
493	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
494	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
495	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
496	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
497	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
498	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
499	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
500	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
501	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
502	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
503	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
504	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
505	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
506	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
507	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
508	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
509	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
510	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
511	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
512	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
513	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
514	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
515	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
516	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
517	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
518	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
519	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
520	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
521	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
522	#else
523	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
524	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
525	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
526	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
527	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
528	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
529	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
530	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
531	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
532	#endif
533	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
534	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
535	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
536	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
537	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
538	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
539	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
540	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
541	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
542	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
543	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
544	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
545	/* 40 UNDEFINED */ hmR0VmxExitPause,
546	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
547	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
548	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
549	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
550	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
551	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitXdtrAccess,
552	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitXdtrAccess,
553	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
554	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
555	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
556	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
557	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
558	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
559	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
560	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
561	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
562	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
563	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
564	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
565	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
566	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
567	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
568	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
569	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
570	};
571	#endif /* HMVMX_USE_FUNCTION_TABLE */
572
573	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
574	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
575	{
576	/* 0 */ "(Not Used)",
577	/* 1 */ "VMCALL executed in VMX root operation.",
578	/* 2 */ "VMCLEAR with invalid physical address.",
579	/* 3 */ "VMCLEAR with VMXON pointer.",
580	/* 4 */ "VMLAUNCH with non-clear VMCS.",
581	/* 5 */ "VMRESUME with non-launched VMCS.",
582	/* 6 */ "VMRESUME after VMXOFF",
583	/* 7 */ "VM-entry with invalid control fields.",
584	/* 8 */ "VM-entry with invalid host state fields.",
585	/* 9 */ "VMPTRLD with invalid physical address.",
586	/* 10 */ "VMPTRLD with VMXON pointer.",
587	/* 11 */ "VMPTRLD with incorrect revision identifier.",
588	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
589	/* 13 */ "VMWRITE to read-only VMCS component.",
590	/* 14 */ "(Not Used)",
591	/* 15 */ "VMXON executed in VMX root operation.",
592	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
593	/* 17 */ "VM-entry with non-launched executing VMCS.",
594	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
595	/* 19 */ "VMCALL with non-clear VMCS.",
596	/* 20 */ "VMCALL with invalid VM-exit control fields.",
597	/* 21 */ "(Not Used)",
598	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
599	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
600	/* 24 */ "VMCALL with invalid SMM-monitor features.",
601	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
602	/* 26 */ "VM-entry with events blocked by MOV SS.",
603	/* 27 */ "(Not Used)",
604	/* 28 */ "Invalid operand to INVEPT/INVVPID."
605	};
606	#endif /* VBOX_STRICT */
607
608
609	/**
610	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
611	*
612	* Any bit set in this mask is owned by the host/hypervisor and would cause a
613	* VM-exit when modified by the guest.
614	*
615	* @returns The static CR0 guest/host mask.
616	* @param pVCpu The cross context virtual CPU structure.
617	*/
618	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PCVMCPU pVCpu)
619	{
620	/*
621	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
622	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
623	*/
624	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
625	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
626	* and @bugref{6944}. */
627	PVM pVM = pVCpu->CTX_SUFF(pVM);
628	return ( X86_CR0_PE
629	\| X86_CR0_NE
630	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
631	\| X86_CR0_PG
632	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
633	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
634	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
635	}
636
637
638	/**
639	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
640	*
641	* Any bit set in this mask is owned by the host/hypervisor and would cause a
642	* VM-exit when modified by the guest.
643	*
644	* @returns The static CR4 guest/host mask.
645	* @param pVCpu The cross context virtual CPU structure.
646	*/
647	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PCVMCPU pVCpu)
648	{
649	/*
650	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
651	* these bits are reserved on hardware that does not support them. Since the
652	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
653	* these bits and handle it depending on whether we expose them to the guest.
654	*/
655	PVM pVM = pVCpu->CTX_SUFF(pVM);
656	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
657	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
658	return ( X86_CR4_VMXE
659	\| X86_CR4_VME
660	\| X86_CR4_PAE
661	\| X86_CR4_PGE
662	\| X86_CR4_PSE
663	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
664	\| (fPcid ? X86_CR4_PCIDE : 0));
665	}
666
667
668	/**
669	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
670	* area.
671	*
672	* @returns @c true if it's different, @c false otherwise.
673	* @param pVmcsInfo The VMCS info. object.
674	*/
675	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
676	{
677	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
678	&& pVmcsInfo->pvGuestMsrStore);
679	}
680
681
682	/**
683	* Adds one or more exceptions to the exception bitmap and commits it to the current
684	* VMCS.
685	*
686	* @returns VBox status code.
687	* @param pVmxTransient The VMX-transient structure.
688	* @param uXcptMask The exception(s) to add.
689	*/
690	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
691	{
692	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
693	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
694	if ((uXcptBitmap & uXcptMask) != uXcptMask)
695	{
696	uXcptBitmap \|= uXcptMask;
697	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
698	AssertRCReturn(rc, rc);
699	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
700	}
701	return VINF_SUCCESS;
702	}
703
704
705	/**
706	* Adds an exception to the exception bitmap and commits it to the current VMCS.
707	*
708	* @returns VBox status code.
709	* @param pVmxTransient The VMX-transient structure.
710	* @param uXcpt The exception to add.
711	*/
712	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
713	{
714	Assert(uXcpt <= X86_XCPT_LAST);
715	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
716	}
717
718
719	/**
720	* Remove one or more exceptions from the exception bitmap and commits it to the
721	* current VMCS.
722	*
723	* This takes care of not removing the exception intercept if a nested-guest
724	* requires the exception to be intercepted.
725	*
726	* @returns VBox status code.
727	* @param pVCpu The cross context virtual CPU structure.
728	* @param pVmxTransient The VMX-transient structure.
729	* @param uXcptMask The exception(s) to remove.
730	*/
731	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
732	{
733	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
734	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
735	if (u32XcptBitmap & uXcptMask)
736	{
737	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
738	if (!pVmxTransient->fIsNestedGuest)
739	{ /* likely */ }
740	else
741	{
742	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
743	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
744	}
745	#endif
746	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
747	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
748	\| RT_BIT(X86_XCPT_DE)
749	\| RT_BIT(X86_XCPT_NM)
750	\| RT_BIT(X86_XCPT_TS)
751	\| RT_BIT(X86_XCPT_UD)
752	\| RT_BIT(X86_XCPT_NP)
753	\| RT_BIT(X86_XCPT_SS)
754	\| RT_BIT(X86_XCPT_GP)
755	\| RT_BIT(X86_XCPT_PF)
756	\| RT_BIT(X86_XCPT_MF));
757	#elif defined(HMVMX_ALWAYS_TRAP_PF)
758	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
759	#endif
760	if (uXcptMask)
761	{
762	/* Validate we are not removing any essential exception intercepts. */
763	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
764	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
765	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
766
767	/* Remove it from the exception bitmap. */
768	u32XcptBitmap &= ~uXcptMask;
769
770	/* Commit and update the cache if necessary. */
771	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
772	{
773	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
774	AssertRCReturn(rc, rc);
775	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
776	}
777	}
778	}
779	return VINF_SUCCESS;
780	}
781
782
783	/**
784	* Remove an exceptions from the exception bitmap and commits it to the current
785	* VMCS.
786	*
787	* @returns VBox status code.
788	* @param pVCpu The cross context virtual CPU structure.
789	* @param pVmxTransient The VMX-transient structure.
790	* @param uXcpt The exception to remove.
791	*/
792	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
793	{
794	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
795	}
796
797
798	/**
799	* Loads the VMCS specified by the VMCS info. object.
800	*
801	* @returns VBox status code.
802	* @param pVmcsInfo The VMCS info. object.
803	*/
804	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
805	{
806	Assert(pVmcsInfo);
807	Assert(pVmcsInfo->HCPhysVmcs);
808	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
809
810	if (pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_CLEAR)
811	{
812	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
813	if (RT_SUCCESS(rc))
814	{
815	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
816	return VINF_SUCCESS;
817	}
818	return rc;
819	}
820	return VERR_VMX_INVALID_VMCS_LAUNCH_STATE;
821	}
822
823
824	/**
825	* Clears the VMCS specified by the VMCS info. object.
826	*
827	* @returns VBox status code.
828	* @param pVmcsInfo The VMCS info. object.
829	*/
830	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
831	{
832	Assert(pVmcsInfo);
833	Assert(pVmcsInfo->HCPhysVmcs);
834	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
835
836	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
837	if (RT_SUCCESS(rc))
838	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
839	return rc;
840	}
841
842
843	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
844	/**
845	* Switches the current VMCS to the one specified.
846	*
847	* @returns VBox status code.
848	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
849	* @param pVmcsInfoTo The VMCS info. object we are switching to.
850	*
851	* @remarks Called with interrupts disabled.
852	*/
853	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
854	{
855	Assert(pVmcsInfoFrom);
856	Assert(pVmcsInfoTo);
857
858	/*
859	* Clear the VMCS we are switching out if it has not already been cleared.
860	* This will sync any CPU internal data back to the VMCS.
861	*/
862	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
863	{
864	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
865	if (RT_SUCCESS(rc))
866	{ /* likely */ }
867	else
868	return rc;
869	}
870
871	/*
872	* Clear the VMCS we are switching to if it has not already been cleared.
873	* This will initialize the VMCS launch state to "clear" required for loading it.
874	*
875	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
876	*/
877	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
878	{
879	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
880	if (RT_SUCCESS(rc))
881	{ /* likely */ }
882	else
883	return rc;
884	}
885
886	/*
887	* Finally, load the VMCS we are switching to.
888	*/
889	return hmR0VmxLoadVmcs(pVmcsInfoTo);
890	}
891	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
892
893
894	/**
895	* Updates the VM's last error record.
896	*
897	* If there was a VMX instruction error, reads the error data from the VMCS and
898	* updates VCPU's last error record as well.
899	*
900	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
901	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
902	* VERR_VMX_INVALID_VMCS_FIELD.
903	* @param rc The error code.
904	*/
905	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
906	{
907	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
908	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
909	{
910	AssertPtrReturnVoid(pVCpu);
911	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
912	}
913	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
914	}
915
916
917	#ifdef VBOX_STRICT
918	/**
919	* Reads the VM-entry interruption-information field from the VMCS into the VMX
920	* transient structure.
921	*
922	* @returns VBox status code.
923	* @param pVmxTransient The VMX-transient structure.
924	*
925	* @remarks No-long-jump zone!!!
926	*/
927	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
928	{
929	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
930	AssertRCReturn(rc, rc);
931	return VINF_SUCCESS;
932	}
933
934
935	/**
936	* Reads the VM-entry exception error code field from the VMCS into
937	* the VMX transient structure.
938	*
939	* @returns VBox status code.
940	* @param pVmxTransient The VMX-transient structure.
941	*
942	* @remarks No-long-jump zone!!!
943	*/
944	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
945	{
946	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
947	AssertRCReturn(rc, rc);
948	return VINF_SUCCESS;
949	}
950
951
952	/**
953	* Reads the VM-entry exception error code field from the VMCS into
954	* the VMX transient structure.
955	*
956	* @returns VBox status code.
957	* @param pVmxTransient The VMX-transient structure.
958	*
959	* @remarks No-long-jump zone!!!
960	*/
961	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
962	{
963	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
964	AssertRCReturn(rc, rc);
965	return VINF_SUCCESS;
966	}
967	#endif /* VBOX_STRICT */
968
969
970	/**
971	* Reads the VM-exit interruption-information field from the VMCS into the VMX
972	* transient structure.
973	*
974	* @returns VBox status code.
975	* @param pVmxTransient The VMX-transient structure.
976	*/
977	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
978	{
979	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
980	{
981	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
982	AssertRCReturn(rc,rc);
983	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
984	}
985	return VINF_SUCCESS;
986	}
987
988
989	/**
990	* Reads the VM-exit interruption error code from the VMCS into the VMX
991	* transient structure.
992	*
993	* @returns VBox status code.
994	* @param pVmxTransient The VMX-transient structure.
995	*/
996	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
997	{
998	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
999	{
1000	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1001	AssertRCReturn(rc, rc);
1002	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1003	}
1004	return VINF_SUCCESS;
1005	}
1006
1007
1008	/**
1009	* Reads the VM-exit instruction length field from the VMCS into the VMX
1010	* transient structure.
1011	*
1012	* @returns VBox status code.
1013	* @param pVmxTransient The VMX-transient structure.
1014	*/
1015	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1016	{
1017	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1018	{
1019	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1020	AssertRCReturn(rc, rc);
1021	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1022	}
1023	return VINF_SUCCESS;
1024	}
1025
1026
1027	/**
1028	* Reads the VM-exit instruction-information field from the VMCS into
1029	* the VMX transient structure.
1030	*
1031	* @returns VBox status code.
1032	* @param pVmxTransient The VMX-transient structure.
1033	*/
1034	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1035	{
1036	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1037	{
1038	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1039	AssertRCReturn(rc, rc);
1040	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1041	}
1042	return VINF_SUCCESS;
1043	}
1044
1045
1046	/**
1047	* Reads the VM-exit Qualification from the VMCS into the VMX transient structure.
1048	*
1049	* @returns VBox status code.
1050	* @param pVCpu The cross context virtual CPU structure of the
1051	* calling EMT. (Required for the VMCS cache case.)
1052	* @param pVmxTransient The VMX-transient structure.
1053	*/
1054	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1055	{
1056	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1057	{
1058	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1059	AssertRCReturn(rc, rc);
1060	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1061	}
1062	return VINF_SUCCESS;
1063	}
1064
1065
1066	/**
1067	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1068	*
1069	* @returns VBox status code.
1070	* @param pVCpu The cross context virtual CPU structure of the
1071	* calling EMT. (Required for the VMCS cache case.)
1072	* @param pVmxTransient The VMX-transient structure.
1073	*/
1074	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1075	{
1076	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1077	{
1078	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1079	AssertRCReturn(rc, rc);
1080	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1081	}
1082	return VINF_SUCCESS;
1083	}
1084
1085
1086	/**
1087	* Reads the IDT-vectoring information field from the VMCS into the VMX
1088	* transient structure.
1089	*
1090	* @returns VBox status code.
1091	* @param pVmxTransient The VMX-transient structure.
1092	*
1093	* @remarks No-long-jump zone!!!
1094	*/
1095	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1096	{
1097	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1098	{
1099	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1100	AssertRCReturn(rc, rc);
1101	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1102	}
1103	return VINF_SUCCESS;
1104	}
1105
1106
1107	/**
1108	* Reads the IDT-vectoring error code from the VMCS into the VMX
1109	* transient structure.
1110	*
1111	* @returns VBox status code.
1112	* @param pVmxTransient The VMX-transient structure.
1113	*/
1114	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1115	{
1116	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1117	{
1118	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1119	AssertRCReturn(rc, rc);
1120	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1121	}
1122	return VINF_SUCCESS;
1123	}
1124
1125
1126	/**
1127	* Enters VMX root mode operation on the current CPU.
1128	*
1129	* @returns VBox status code.
1130	* @param pVM The cross context VM structure. Can be
1131	* NULL, after a resume.
1132	* @param HCPhysCpuPage Physical address of the VMXON region.
1133	* @param pvCpuPage Pointer to the VMXON region.
1134	*/
1135	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1136	{
1137	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1138	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1139	Assert(pvCpuPage);
1140	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1141
1142	if (pVM)
1143	{
1144	/* Write the VMCS revision identifier to the VMXON region. */
1145	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1146	}
1147
1148	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1149	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1150
1151	/* Enable the VMX bit in CR4 if necessary. */
1152	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1153
1154	/* Enter VMX root mode. */
1155	int rc = VMXEnable(HCPhysCpuPage);
1156	if (RT_FAILURE(rc))
1157	{
1158	if (!(uOldCr4 & X86_CR4_VMXE))
1159	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1160
1161	if (pVM)
1162	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1163	}
1164
1165	/* Restore interrupts. */
1166	ASMSetFlags(fEFlags);
1167	return rc;
1168	}
1169
1170
1171	/**
1172	* Exits VMX root mode operation on the current CPU.
1173	*
1174	* @returns VBox status code.
1175	*/
1176	static int hmR0VmxLeaveRootMode(void)
1177	{
1178	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1179
1180	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1181	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1182
1183	/* If we're for some reason not in VMX root mode, then don't leave it. */
1184	RTCCUINTREG const uHostCR4 = ASMGetCR4();
1185
1186	int rc;
1187	if (uHostCR4 & X86_CR4_VMXE)
1188	{
1189	/* Exit VMX root mode and clear the VMX bit in CR4. */
1190	VMXDisable();
1191	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1192	rc = VINF_SUCCESS;
1193	}
1194	else
1195	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1196
1197	/* Restore interrupts. */
1198	ASMSetFlags(fEFlags);
1199	return rc;
1200	}
1201
1202
1203	/**
1204	* Allocates and maps a physically contiguous page. The allocated page is
1205	* zero'd out (used by various VT-x structures).
1206	*
1207	* @returns IPRT status code.
1208	* @param pMemObj Pointer to the ring-0 memory object.
1209	* @param ppVirt Where to store the virtual address of the
1210	* allocation.
1211	* @param pHCPhys Where to store the physical address of the
1212	* allocation.
1213	*/
1214	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1215	{
1216	AssertPtr(pMemObj);
1217	AssertPtr(ppVirt);
1218	AssertPtr(pHCPhys);
1219	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1220	if (RT_FAILURE(rc))
1221	return rc;
1222	ppVirt = RTR0MemObjAddress(pMemObj);
1223	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1224	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1225	return VINF_SUCCESS;
1226	}
1227
1228
1229	/**
1230	* Frees and unmaps an allocated, physical page.
1231	*
1232	* @param pMemObj Pointer to the ring-0 memory object.
1233	* @param ppVirt Where to re-initialize the virtual address of
1234	* allocation as 0.
1235	* @param pHCPhys Where to re-initialize the physical address of the
1236	* allocation as 0.
1237	*/
1238	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1239	{
1240	AssertPtr(pMemObj);
1241	AssertPtr(ppVirt);
1242	AssertPtr(pHCPhys);
1243	/* NULL is valid, accepted and ignored by the free function below. */
1244	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1245	*pMemObj = NIL_RTR0MEMOBJ;
1246	*ppVirt = NULL;
1247	*pHCPhys = NIL_RTHCPHYS;
1248	}
1249
1250
1251	/**
1252	* Initializes a VMCS info. object.
1253	*
1254	* @param pVmcsInfo The VMCS info. object.
1255	*/
1256	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1257	{
1258	RT_ZERO(*pVmcsInfo);
1259
1260	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1261	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1262	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1263	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1264	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1265	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1266	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1267	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1268	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1269	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1270	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1271	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1272	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1273	}
1274
1275
1276	/**
1277	* Frees the VT-x structures for a VMCS info. object.
1278	*
1279	* @param pVM The cross context VM structure.
1280	* @param pVmcsInfo The VMCS info. object.
1281	*/
1282	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1283	{
1284	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1285
1286	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1287	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1288
1289	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1290	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1291	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1292
1293	hmR0VmxInitVmcsInfo(pVmcsInfo);
1294	}
1295
1296
1297	/**
1298	* Allocates the VT-x structures for a VMCS info. object.
1299	*
1300	* @returns VBox status code.
1301	* @param pVCpu The cross context virtual CPU structure.
1302	* @param pVmcsInfo The VMCS info. object.
1303	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1304	*/
1305	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1306	{
1307	PVM pVM = pVCpu->CTX_SUFF(pVM);
1308
1309	/* Allocate the guest VM control structure (VMCS). */
1310	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1311	if (RT_SUCCESS(rc))
1312	{
1313	if (!fIsNstGstVmcs)
1314	{
1315	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1316	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1317	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1318	{
1319	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1320	NULL /* pR3Ptr /, NULL / pRCPtr */);
1321	}
1322	}
1323	else
1324	{
1325	Assert(pVmcsInfo->HCPhysVirtApic == NIL_RTHCPHYS);
1326	Assert(!pVmcsInfo->pbVirtApic);
1327	}
1328
1329	if (RT_SUCCESS(rc))
1330	{
1331	/*
1332	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1333	* transparent accesses of specific MSRs.
1334	*
1335	* If the condition for enabling MSR bitmaps changes here, don't forget to
1336	* update HMIsMsrBitmapActive().
1337	*
1338	* We don't share MSR bitmaps between the guest and nested-guest as we then
1339	* don't need to care about carefully restoring the guest MSR bitmap.
1340	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1341	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1342	*/
1343	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1344	{
1345	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1346	if (RT_SUCCESS(rc))
1347	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1348	}
1349
1350	if (RT_SUCCESS(rc))
1351	{
1352	/*
1353	* Allocate the VM-entry MSR-load area for the guest MSRs.
1354	*
1355	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1356	* the guest and nested-guest.
1357	*/
1358	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1359	&pVmcsInfo->HCPhysGuestMsrLoad);
1360	if (RT_SUCCESS(rc))
1361	{
1362	/*
1363	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1364	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1365	*/
1366	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1367	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1368	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1369
1370	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1371	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1372	&pVmcsInfo->HCPhysHostMsrLoad);
1373	}
1374	}
1375	}
1376	}
1377
1378	return rc;
1379	}
1380
1381
1382	/**
1383	* Free all VT-x structures for the VM.
1384	*
1385	* @returns IPRT status code.
1386	* @param pVM The cross context VM structure.
1387	*/
1388	static void hmR0VmxStructsFree(PVM pVM)
1389	{
1390	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1391	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1392	#endif
1393	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1394
1395	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1396	{
1397	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1398	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1399	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1400	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1401	if (pVM->cpum.ro.GuestFeatures.fVmx)
1402	{
1403	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1404	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1405	}
1406	#endif
1407	}
1408	}
1409
1410
1411	/**
1412	* Allocate all VT-x structures for the VM.
1413	*
1414	* @returns IPRT status code.
1415	* @param pVM The cross context VM structure.
1416	*/
1417	static int hmR0VmxStructsAlloc(PVM pVM)
1418	{
1419	/*
1420	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1421	* The VMCS size cannot be more than 4096 bytes.
1422	*
1423	* See Intel spec. Appendix A.1 "Basic VMX Information".
1424	*/
1425	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1426	if (cbVmcs <= X86_PAGE_4K_SIZE)
1427	{ /* likely */ }
1428	else
1429	{
1430	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1431	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1432	}
1433
1434	/*
1435	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1436	*/
1437	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1438	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1439	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1440	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1441	#endif
1442
1443	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1444	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1445	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1446
1447	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1448	{
1449	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1450	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1451	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1452	}
1453
1454	/*
1455	* Allocate per-VM VT-x structures.
1456	*/
1457	int rc = VINF_SUCCESS;
1458	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1459	/* Allocate crash-dump magic scratch page. */
1460	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1461	if (RT_FAILURE(rc))
1462	{
1463	hmR0VmxStructsFree(pVM);
1464	return rc;
1465	}
1466	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1467	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1468	#endif
1469
1470	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1471	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1472	{
1473	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1474	&pVM->hm.s.vmx.HCPhysApicAccess);
1475	if (RT_FAILURE(rc))
1476	{
1477	hmR0VmxStructsFree(pVM);
1478	return rc;
1479	}
1480	}
1481
1482	/*
1483	* Initialize per-VCPU VT-x structures.
1484	*/
1485	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1486	{
1487	/* Allocate the guest VMCS structures. */
1488	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1489	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
1490	if (RT_SUCCESS(rc))
1491	{
1492	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1493	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
1494	if (pVM->cpum.ro.GuestFeatures.fVmx)
1495	{
1496	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
1497	if (RT_SUCCESS(rc))
1498	{ /* likely */ }
1499	else
1500	break;
1501	}
1502	#endif
1503	}
1504	else
1505	break;
1506	}
1507
1508	if (RT_FAILURE(rc))
1509	{
1510	hmR0VmxStructsFree(pVM);
1511	return rc;
1512	}
1513
1514	return VINF_SUCCESS;
1515	}
1516
1517
1518	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1519	/**
1520	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
1521	*
1522	* @returns @c true if the MSR is intercepted, @c false otherwise.
1523	* @param pvMsrBitmap The MSR bitmap.
1524	* @param offMsr The MSR byte offset.
1525	* @param iBit The bit offset from the byte offset.
1526	*/
1527	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
1528	{
1529	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
1530	Assert(pbMsrBitmap);
1531	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
1532	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
1533	}
1534	#endif
1535
1536
1537	/**
1538	* Sets the permission bits for the specified MSR in the given MSR bitmap.
1539	*
1540	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
1541	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
1542	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
1543	* the read/write access of this MSR.
1544	*
1545	* @param pVCpu The cross context virtual CPU structure.
1546	* @param pVmcsInfo The VMCS info. object.
1547	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1548	* @param idMsr The MSR value.
1549	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
1550	* include both a read -and- a write permission!
1551	*
1552	* @sa HMGetVmxMsrPermission.
1553	*/
1554	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
1555	{
1556	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
1557	Assert(pbMsrBitmap);
1558	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
1559
1560	/*
1561	* MSR-bitmap Layout:
1562	* Byte index MSR range Interpreted as
1563	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1564	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1565	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1566	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1567	*
1568	* A bit corresponding to an MSR within the above range causes a VM-exit
1569	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
1570	* the MSR range, it always cause a VM-exit.
1571	*
1572	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
1573	*/
1574	uint16_t const offBitmapRead = 0;
1575	uint16_t const offBitmapWrite = 0x800;
1576	uint16_t offMsr;
1577	int32_t iBit;
1578	if (idMsr <= UINT32_C(0x00001fff))
1579	{
1580	offMsr = 0;
1581	iBit = idMsr;
1582	}
1583	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1584	{
1585	offMsr = 0x400;
1586	iBit = idMsr - UINT32_C(0xc0000000);
1587	}
1588	else
1589	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
1590
1591	/*
1592	* Set the MSR read permission.
1593	*/
1594	uint16_t const offMsrRead = offBitmapRead + offMsr;
1595	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
1596	if (fMsrpm & VMXMSRPM_ALLOW_RD)
1597	{
1598	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1599	bool const fClear = !fIsNstGstVmcs ? true
1600	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
1601	#else
1602	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1603	bool const fClear = true;
1604	#endif
1605	if (fClear)
1606	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
1607	}
1608	else
1609	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
1610
1611	/*
1612	* Set the MSR write permission.
1613	*/
1614	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
1615	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
1616	if (fMsrpm & VMXMSRPM_ALLOW_WR)
1617	{
1618	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1619	bool const fClear = !fIsNstGstVmcs ? true
1620	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
1621	#else
1622	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1623	bool const fClear = true;
1624	#endif
1625	if (fClear)
1626	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
1627	}
1628	else
1629	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
1630	}
1631
1632
1633	/**
1634	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1635	* area.
1636	*
1637	* @returns VBox status code.
1638	* @param pVCpu The cross context virtual CPU structure.
1639	* @param pVmcsInfo The VMCS info. object.
1640	* @param cMsrs The number of MSRs.
1641	*/
1642	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
1643	{
1644	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1645	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1646	if (RT_UNLIKELY(cMsrs >= cMaxSupportedMsrs))
1647	{
1648	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1649	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1650	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1651	}
1652
1653	/* Commit the MSR counts to the VMCS and update the cache. */
1654	int rc = VINF_SUCCESS;
1655	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
1656	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1657	if (pVmcsInfo->cExitMsrStore != cMsrs)
1658	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1659	if (pVmcsInfo->cExitMsrLoad != cMsrs)
1660	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1661	AssertRCReturn(rc, rc);
1662
1663	pVmcsInfo->cEntryMsrLoad = cMsrs;
1664	pVmcsInfo->cExitMsrStore = cMsrs;
1665	pVmcsInfo->cExitMsrLoad = cMsrs;
1666
1667	return VINF_SUCCESS;
1668	}
1669
1670
1671	/**
1672	* Adds a new (or updates the value of an existing) guest/host MSR
1673	* pair to be swapped during the world-switch as part of the
1674	* auto-load/store MSR area in the VMCS.
1675	*
1676	* @returns VBox status code.
1677	* @param pVCpu The cross context virtual CPU structure.
1678	* @param pVmxTransient The VMX-transient structure.
1679	* @param idMsr The MSR.
1680	* @param uGuestMsrValue Value of the guest MSR.
1681	* @param fSetReadWrite Whether to set the guest read/write access of this
1682	* MSR (thus not causing a VM-exit).
1683	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1684	* necessary.
1685	*/
1686	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
1687	bool fSetReadWrite, bool fUpdateHostMsr)
1688	{
1689	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1690	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1691	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1692	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1693	uint32_t i;
1694
1695	/* Paranoia. */
1696	Assert(pGuestMsrLoad);
1697
1698	/* Check if the MSR already exists in the VM-entry MSR-load area. */
1699	for (i = 0; i < cMsrs; i++)
1700	{
1701	if (pGuestMsrLoad->u32Msr == idMsr)
1702	break;
1703	pGuestMsrLoad++;
1704	}
1705
1706	bool fAdded = false;
1707	if (i == cMsrs)
1708	{
1709	/* The MSR does not exist, bump the MSR coun to make room for the new MSR. */
1710	++cMsrs;
1711	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1712	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
1713
1714	/* Set the guest to read/write this MSR without causing VM-exits. */
1715	if ( fSetReadWrite
1716	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
1717	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
1718
1719	fAdded = true;
1720	}
1721
1722	/* Update the MSR value for the newly added or already existing MSR. */
1723	pGuestMsrLoad->u32Msr = idMsr;
1724	pGuestMsrLoad->u64Value = uGuestMsrValue;
1725
1726	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
1727	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1728	{
1729	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1730	pGuestMsrStore += i;
1731	pGuestMsrStore->u32Msr = idMsr;
1732	pGuestMsrStore->u64Value = 0;
1733	}
1734
1735	/* Update the corresponding slot in the host MSR area. */
1736	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1737	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad && pHostMsr != pVmcsInfo->pvGuestMsrStore);
1738	pHostMsr += i;
1739	pHostMsr->u32Msr = idMsr;
1740
1741	/*
1742	* Only if the caller requests to update the host MSR value AND we've newly added the
1743	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
1744	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
1745	*
1746	* We do this for performance reasons since reading MSRs may be quite expensive.
1747	*/
1748	if ( fAdded
1749	&& fUpdateHostMsr)
1750	{
1751	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1752	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1753	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1754	}
1755	return VINF_SUCCESS;
1756	}
1757
1758
1759	/**
1760	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1761	* auto-load/store MSR area in the VMCS.
1762	*
1763	* @returns VBox status code.
1764	* @param pVCpu The cross context virtual CPU structure.
1765	* @param pVmxTransient The VMX-transient structure.
1766	* @param idMsr The MSR.
1767	*/
1768	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
1769	{
1770	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1771	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1772	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1773	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1774
1775	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
1776	for (uint32_t i = 0; i < cMsrs; i++)
1777	{
1778	/* Find the MSR. */
1779	if (pGuestMsrLoad->u32Msr == idMsr)
1780	{
1781	/* If it's the last MSR, simply reduce the count. */
1782	if (i == cMsrs - 1)
1783	{
1784	--cMsrs;
1785	break;
1786	}
1787
1788	/* Remove it by copying the last MSR in place of it, and reducing the count. */
1789	PVMXAUTOMSR pLastGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1790	pLastGuestMsrLoad += cMsrs - 1;
1791	pGuestMsrLoad->u32Msr = pLastGuestMsrLoad->u32Msr;
1792	pGuestMsrLoad->u64Value = pLastGuestMsrLoad->u64Value;
1793
1794	/* Remove it from the VM-exit MSR-store area if we are using a different page. */
1795	if (fSeparateExitMsrStorePage)
1796	{
1797	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1798	PVMXAUTOMSR pLastGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1799	pGuestMsrStore += i;
1800	pLastGuestMsrStore += cMsrs - 1;
1801	Assert(pGuestMsrStore->u32Msr == idMsr);
1802	pGuestMsrStore->u32Msr = pLastGuestMsrStore->u32Msr;
1803	pGuestMsrStore->u64Value = pLastGuestMsrStore->u64Value;
1804	}
1805
1806	/* Remove it from the VM-exit MSR-load area. */
1807	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1808	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1809	pHostMsr += i;
1810	pLastHostMsr += cMsrs - 1;
1811	Assert(pHostMsr->u32Msr == idMsr);
1812	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1813	pHostMsr->u64Value = pLastHostMsr->u64Value;
1814	--cMsrs;
1815	break;
1816	}
1817	pGuestMsrLoad++;
1818	}
1819
1820	/* Update the VMCS if the count changed (meaning the MSR was found). */
1821	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
1822	{
1823	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1824	AssertRCReturn(rc, rc);
1825
1826	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1827	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1828	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
1829
1830	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
1831	return VINF_SUCCESS;
1832	}
1833
1834	return VERR_NOT_FOUND;
1835	}
1836
1837
1838	/**
1839	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
1840	*
1841	* @returns @c true if found, @c false otherwise.
1842	* @param pVmcsInfo The VMCS info. object.
1843	* @param idMsr The MSR to find.
1844	*/
1845	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
1846	{
1847	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1848	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
1849	for (uint32_t i = 0; i < cMsrs; i++)
1850	{
1851	if (pGuestMsrLoad->u32Msr == idMsr)
1852	return true;
1853	pGuestMsrLoad++;
1854	}
1855	return false;
1856	}
1857
1858
1859	/**
1860	* Updates the value of all host MSRs in the VM-exit MSR-load area.
1861	*
1862	* @param pVCpu The cross context virtual CPU structure.
1863	* @param pVmcsInfo The VMCS info. object.
1864	*
1865	* @remarks No-long-jump zone!!!
1866	*/
1867	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
1868	{
1869	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1870	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
1871
1872	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1873	Assert(pHostMsrLoad);
1874
1875	for (uint32_t i = 0; i < cMsrs; i++, pHostMsrLoad++)
1876	{
1877	/*
1878	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1879	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1880	*/
1881	if (pHostMsrLoad->u32Msr == MSR_K6_EFER)
1882	pHostMsrLoad->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
1883	else
1884	pHostMsrLoad->u64Value = ASMRdMsr(pHostMsrLoad->u32Msr);
1885	}
1886	}
1887
1888
1889	/**
1890	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1891	* perform lazy restoration of the host MSRs while leaving VT-x.
1892	*
1893	* @param pVCpu The cross context virtual CPU structure.
1894	*
1895	* @remarks No-long-jump zone!!!
1896	*/
1897	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1898	{
1899	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1900
1901	/*
1902	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
1903	*/
1904	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1905	{
1906	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1907	#if HC_ARCH_BITS == 64
1908	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1909	{
1910	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
1911	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
1912	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
1913	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1914	}
1915	#endif
1916	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1917	}
1918	}
1919
1920
1921	/**
1922	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1923	* lazily while leaving VT-x.
1924	*
1925	* @returns true if it does, false otherwise.
1926	* @param pVCpu The cross context virtual CPU structure.
1927	* @param idMsr The MSR to check.
1928	*/
1929	static bool hmR0VmxIsLazyGuestMsr(PCVMCPU pVCpu, uint32_t idMsr)
1930	{
1931	NOREF(pVCpu);
1932	#if HC_ARCH_BITS == 64
1933	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1934	{
1935	switch (idMsr)
1936	{
1937	case MSR_K8_LSTAR:
1938	case MSR_K6_STAR:
1939	case MSR_K8_SF_MASK:
1940	case MSR_K8_KERNEL_GS_BASE:
1941	return true;
1942	}
1943	}
1944	#else
1945	RT_NOREF(pVCpu, idMsr);
1946	#endif
1947	return false;
1948	}
1949
1950
1951	/**
1952	* Loads a set of guests MSRs to allow read/passthru to the guest.
1953	*
1954	* The name of this function is slightly confusing. This function does NOT
1955	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1956	* common prefix for functions dealing with "lazy restoration" of the shared
1957	* MSRs.
1958	*
1959	* @param pVCpu The cross context virtual CPU structure.
1960	*
1961	* @remarks No-long-jump zone!!!
1962	*/
1963	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1964	{
1965	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1966	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1967
1968	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1969	#if HC_ARCH_BITS == 64
1970	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1971	{
1972	/*
1973	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1974	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1975	* we can skip a few MSR writes.
1976	*
1977	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1978	* guest MSR values in the guest-CPU context might be different to what's currently
1979	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1980	* CPU, see @bugref{8728}.
1981	*/
1982	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1983	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1984	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
1985	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
1986	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
1987	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
1988	{
1989	#ifdef VBOX_STRICT
1990	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1991	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1992	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1993	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1994	#endif
1995	}
1996	else
1997	{
1998	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1999	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2000	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2001	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2002	}
2003	}
2004	#endif
2005	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2006	}
2007
2008
2009	/**
2010	* Performs lazy restoration of the set of host MSRs if they were previously
2011	* loaded with guest MSR values.
2012	*
2013	* @param pVCpu The cross context virtual CPU structure.
2014	*
2015	* @remarks No-long-jump zone!!!
2016	* @remarks The guest MSRs should have been saved back into the guest-CPU
2017	* context by hmR0VmxImportGuestState()!!!
2018	*/
2019	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2020	{
2021	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2022	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2023
2024	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2025	{
2026	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2027	#if HC_ARCH_BITS == 64
2028	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2029	{
2030	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2031	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2032	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2033	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2034	}
2035	#endif
2036	}
2037	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2038	}
2039
2040
2041	/**
2042	* Verifies that our cached values of the VMCS fields are all consistent with
2043	* what's actually present in the VMCS.
2044	*
2045	* @returns VBox status code.
2046	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2047	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2048	* VMCS content. HMCPU error-field is
2049	* updated, see VMX_VCI_XXX.
2050	* @param pVCpu The cross context virtual CPU structure.
2051	* @param pVmcsInfo The VMCS info. object.
2052	*/
2053	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2054	{
2055	uint32_t u32Val;
2056	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2057	AssertRCReturn(rc, rc);
2058	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2059	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32EntryCtls, u32Val),
2060	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2061	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2062
2063	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2064	AssertRCReturn(rc, rc);
2065	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2066	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ExitCtls, u32Val),
2067	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2068	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2069
2070	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2071	AssertRCReturn(rc, rc);
2072	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2073	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32PinCtls, u32Val),
2074	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2075	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2076
2077	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2078	AssertRCReturn(rc, rc);
2079	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2080	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls, u32Val),
2081	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2082	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2083
2084	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2085	{
2086	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2087	AssertRCReturn(rc, rc);
2088	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2089	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls2, u32Val),
2090	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2091	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2092	}
2093
2094	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2095	AssertRCReturn(rc, rc);
2096	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2097	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32XcptBitmap, u32Val),
2098	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2099	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2100
2101	uint64_t u64Val;
2102	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2103	AssertRCReturn(rc, rc);
2104	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2105	("Cache=%#RX64 VMCS=%#RX64\n", pVmcsInfo->u64TscOffset, u64Val),
2106	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2107	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2108
2109	return VINF_SUCCESS;
2110	}
2111
2112
2113	#ifdef VBOX_STRICT
2114	/**
2115	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2116	*
2117	* @param pVCpu The cross context virtual CPU structure.
2118	* @param pVmcsInfo The VMCS info. object.
2119	*/
2120	static void hmR0VmxCheckHostEferMsr(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2121	{
2122	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2123
2124	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2125	{
2126	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2127	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2128	uint64_t uVmcsEferMsrVmcs;
2129	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2130	AssertRC(rc);
2131
2132	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2133	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2134	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2135	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2136	}
2137	}
2138
2139
2140	/**
2141	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2142	* VMCS are correct.
2143	*
2144	* @param pVCpu The cross context virtual CPU structure.
2145	* @param pVmcsInfo The VMCS info. object.
2146	*/
2147	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2148	{
2149	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2150
2151	/* Verify MSR counts in the VMCS are what we think it should be. */
2152	uint32_t cMsrs;
2153	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs);
2154	AssertRC(rc);
2155	Assert(cMsrs == pVmcsInfo->cEntryMsrLoad);
2156
2157	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs);
2158	AssertRC(rc);
2159	Assert(cMsrs == pVmcsInfo->cExitMsrStore);
2160
2161	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs);
2162	AssertRC(rc);
2163	Assert(cMsrs == pVmcsInfo->cExitMsrLoad);
2164
2165	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2166	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2167
2168	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2169	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2170	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2171	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2172	for (uint32_t i = 0; i < cMsrs; i++)
2173	{
2174	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2175	if (fSeparateExitMsrStorePage)
2176	{
2177	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2178	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2179	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2180	}
2181
2182	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2183	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2184	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2185
2186	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2187	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2188	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2189	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2190
2191	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2192	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2193	{
2194	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2195	if (pGuestMsrLoad->u32Msr == MSR_K6_EFER)
2196	{
2197	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2198	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2199	}
2200	else
2201	{
2202	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2203	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2204	}
2205	}
2206
2207	/* Move to the next MSR. */
2208	pHostMsrLoad++;
2209	pGuestMsrLoad++;
2210	pGuestMsrStore++;
2211	}
2212	}
2213	#endif /* VBOX_STRICT */
2214
2215
2216	/**
2217	* Flushes the TLB using EPT.
2218	*
2219	* @returns VBox status code.
2220	* @param pVCpu The cross context virtual CPU structure of the calling
2221	* EMT. Can be NULL depending on @a enmTlbFlush.
2222	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2223	* enmTlbFlush.
2224	* @param enmTlbFlush Type of flush.
2225	*
2226	* @remarks Caller is responsible for making sure this function is called only
2227	* when NestedPaging is supported and providing @a enmTlbFlush that is
2228	* supported by the CPU.
2229	* @remarks Can be called with interrupts disabled.
2230	*/
2231	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2232	{
2233	uint64_t au64Descriptor[2];
2234	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2235	au64Descriptor[0] = 0;
2236	else
2237	{
2238	Assert(pVCpu);
2239	Assert(pVmcsInfo);
2240	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2241	}
2242	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2243
2244	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2245	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2246
2247	if ( RT_SUCCESS(rc)
2248	&& pVCpu)
2249	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2250	}
2251
2252
2253	/**
2254	* Flushes the TLB using VPID.
2255	*
2256	* @returns VBox status code.
2257	* @param pVCpu The cross context virtual CPU structure of the calling
2258	* EMT. Can be NULL depending on @a enmTlbFlush.
2259	* @param enmTlbFlush Type of flush.
2260	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2261	* on @a enmTlbFlush).
2262	*
2263	* @remarks Can be called with interrupts disabled.
2264	*/
2265	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2266	{
2267	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2268
2269	uint64_t au64Descriptor[2];
2270	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2271	{
2272	au64Descriptor[0] = 0;
2273	au64Descriptor[1] = 0;
2274	}
2275	else
2276	{
2277	AssertPtr(pVCpu);
2278	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2279	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2280	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2281	au64Descriptor[1] = GCPtr;
2282	}
2283
2284	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2285	AssertMsg(rc == VINF_SUCCESS,
2286	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2287
2288	if ( RT_SUCCESS(rc)
2289	&& pVCpu)
2290	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2291	NOREF(rc);
2292	}
2293
2294
2295	/**
2296	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2297	* otherwise there is nothing really to invalidate.
2298	*
2299	* @returns VBox status code.
2300	* @param pVCpu The cross context virtual CPU structure.
2301	* @param GCVirt Guest virtual address of the page to invalidate.
2302	*/
2303	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2304	{
2305	AssertPtr(pVCpu);
2306	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2307
2308	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2309	{
2310	/*
2311	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2312	* the EPT case. See @bugref{6043} and @bugref{6177}.
2313	*
2314	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2315	* as this function maybe called in a loop with individual addresses.
2316	*/
2317	PVM pVM = pVCpu->CTX_SUFF(pVM);
2318	if (pVM->hm.s.vmx.fVpid)
2319	{
2320	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2321
2322	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2323	/*
2324	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2325	* where executing INVVPID outside 64-bit mode does not flush translations of
2326	* 64-bit linear addresses, see @bugref{6208#c72}.
2327	*/
2328	if (RT_HI_U32(GCVirt))
2329	fVpidFlush = false;
2330	#endif
2331
2332	if (fVpidFlush)
2333	{
2334	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2335	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2336	}
2337	else
2338	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2339	}
2340	else if (pVM->hm.s.fNestedPaging)
2341	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2342	}
2343
2344	return VINF_SUCCESS;
2345	}
2346
2347
2348	/**
2349	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2350	* case where neither EPT nor VPID is supported by the CPU.
2351	*
2352	* @param pHostCpu The HM physical-CPU structure.
2353	* @param pVCpu The cross context virtual CPU structure.
2354	*
2355	* @remarks Called with interrupts disabled.
2356	*/
2357	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2358	{
2359	AssertPtr(pVCpu);
2360	AssertPtr(pHostCpu);
2361
2362	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2363
2364	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2365	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2366	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2367	pVCpu->hm.s.fForceTLBFlush = false;
2368	return;
2369	}
2370
2371
2372	/**
2373	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2374	*
2375	* @param pHostCpu The HM physical-CPU structure.
2376	* @param pVCpu The cross context virtual CPU structure.
2377	* @param pVmcsInfo The VMCS info. object.
2378	*
2379	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2380	* nomenclature. The reason is, to avoid confusion in compare statements
2381	* since the host-CPU copies are named "ASID".
2382	*
2383	* @remarks Called with interrupts disabled.
2384	*/
2385	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2386	{
2387	#ifdef VBOX_WITH_STATISTICS
2388	bool fTlbFlushed = false;
2389	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2390	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2391	if (!fTlbFlushed) \
2392	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2393	} while (0)
2394	#else
2395	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2396	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2397	#endif
2398
2399	AssertPtr(pVCpu);
2400	AssertPtr(pHostCpu);
2401	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2402
2403	PVM pVM = pVCpu->CTX_SUFF(pVM);
2404	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2405	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2406	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2407
2408	/*
2409	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2410	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2411	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2412	* cannot reuse the current ASID anymore.
2413	*/
2414	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2415	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2416	{
2417	++pHostCpu->uCurrentAsid;
2418	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2419	{
2420	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2421	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2422	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2423	}
2424
2425	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2426	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2427	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2428
2429	/*
2430	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2431	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2432	*/
2433	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2434	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2435	HMVMX_SET_TAGGED_TLB_FLUSHED();
2436	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2437	}
2438	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2439	{
2440	/*
2441	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2442	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2443	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2444	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2445	* mappings, see @bugref{6568}.
2446	*
2447	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2448	*/
2449	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2450	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2451	HMVMX_SET_TAGGED_TLB_FLUSHED();
2452	}
2453
2454	pVCpu->hm.s.fForceTLBFlush = false;
2455	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2456
2457	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
2458	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
2459	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2460	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2461	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2462	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2463	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2464	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2465	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2466
2467	/* Update VMCS with the VPID. */
2468	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2469	AssertRC(rc);
2470
2471	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2472	}
2473
2474
2475	/**
2476	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2477	*
2478	* @param pHostCpu The HM physical-CPU structure.
2479	* @param pVCpu The cross context virtual CPU structure.
2480	* @param pVmcsInfo The VMCS info. object.
2481	*
2482	* @remarks Called with interrupts disabled.
2483	*/
2484	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2485	{
2486	AssertPtr(pVCpu);
2487	AssertPtr(pHostCpu);
2488	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2489	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2490	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2491
2492	/*
2493	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2494	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2495	*/
2496	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2497	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2498	{
2499	pVCpu->hm.s.fForceTLBFlush = true;
2500	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2501	}
2502
2503	/* Check for explicit TLB flushes. */
2504	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2505	{
2506	pVCpu->hm.s.fForceTLBFlush = true;
2507	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2508	}
2509
2510	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2511	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2512
2513	if (pVCpu->hm.s.fForceTLBFlush)
2514	{
2515	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2516	pVCpu->hm.s.fForceTLBFlush = false;
2517	}
2518	}
2519
2520
2521	/**
2522	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2523	*
2524	* @param pHostCpu The HM physical-CPU structure.
2525	* @param pVCpu The cross context virtual CPU structure.
2526	*
2527	* @remarks Called with interrupts disabled.
2528	*/
2529	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2530	{
2531	AssertPtr(pVCpu);
2532	AssertPtr(pHostCpu);
2533	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2534	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2535	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2536
2537	/*
2538	* Force a TLB flush for the first world switch if the current CPU differs from the one we
2539	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2540	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2541	* cannot reuse the current ASID anymore.
2542	*/
2543	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2544	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2545	{
2546	pVCpu->hm.s.fForceTLBFlush = true;
2547	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2548	}
2549
2550	/* Check for explicit TLB flushes. */
2551	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2552	{
2553	/*
2554	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2555	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2556	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
2557	* include fExplicitFlush's too) - an obscure corner case.
2558	*/
2559	pVCpu->hm.s.fForceTLBFlush = true;
2560	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2561	}
2562
2563	PVM pVM = pVCpu->CTX_SUFF(pVM);
2564	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2565	if (pVCpu->hm.s.fForceTLBFlush)
2566	{
2567	++pHostCpu->uCurrentAsid;
2568	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2569	{
2570	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2571	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2572	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2573	}
2574
2575	pVCpu->hm.s.fForceTLBFlush = false;
2576	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2577	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2578	if (pHostCpu->fFlushAsidBeforeUse)
2579	{
2580	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2581	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2582	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2583	{
2584	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2585	pHostCpu->fFlushAsidBeforeUse = false;
2586	}
2587	else
2588	{
2589	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2590	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2591	}
2592	}
2593	}
2594
2595	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2596	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2597	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2598	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2599	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2600	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2601	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2602
2603	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2604	AssertRC(rc);
2605	}
2606
2607
2608	/**
2609	* Flushes the guest TLB entry based on CPU capabilities.
2610	*
2611	* @param pHostCpu The HM physical-CPU structure.
2612	* @param pVCpu The cross context virtual CPU structure.
2613	* @param pVmcsInfo The VMCS info. object.
2614	*
2615	* @remarks Called with interrupts disabled.
2616	*/
2617	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2618	{
2619	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2620	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2621	#endif
2622	PVM pVM = pVCpu->CTX_SUFF(pVM);
2623	switch (pVM->hm.s.vmx.enmTlbFlushType)
2624	{
2625	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
2626	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
2627	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
2628	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
2629	default:
2630	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2631	break;
2632	}
2633	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2634	}
2635
2636
2637	/**
2638	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2639	* TLB entries from the host TLB before VM-entry.
2640	*
2641	* @returns VBox status code.
2642	* @param pVM The cross context VM structure.
2643	*/
2644	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2645	{
2646	/*
2647	* Determine optimal flush type for nested paging.
2648	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
2649	* unrestricted guest execution (see hmR3InitFinalizeR0()).
2650	*/
2651	if (pVM->hm.s.fNestedPaging)
2652	{
2653	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2654	{
2655	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2656	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2657	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2658	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2659	else
2660	{
2661	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2662	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2663	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2664	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2665	}
2666
2667	/* Make sure the write-back cacheable memory type for EPT is supported. */
2668	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2669	{
2670	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2671	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2672	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2673	}
2674
2675	/* EPT requires a page-walk length of 4. */
2676	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2677	{
2678	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2679	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2680	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2681	}
2682	}
2683	else
2684	{
2685	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2686	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2687	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2688	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2689	}
2690	}
2691
2692	/*
2693	* Determine optimal flush type for VPID.
2694	*/
2695	if (pVM->hm.s.vmx.fVpid)
2696	{
2697	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2698	{
2699	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2700	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2701	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2702	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2703	else
2704	{
2705	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2706	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2707	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2708	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2709	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2710	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2711	pVM->hm.s.vmx.fVpid = false;
2712	}
2713	}
2714	else
2715	{
2716	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2717	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2718	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2719	pVM->hm.s.vmx.fVpid = false;
2720	}
2721	}
2722
2723	/*
2724	* Setup the handler for flushing tagged-TLBs.
2725	*/
2726	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2727	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2728	else if (pVM->hm.s.fNestedPaging)
2729	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2730	else if (pVM->hm.s.vmx.fVpid)
2731	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2732	else
2733	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2734	return VINF_SUCCESS;
2735	}
2736
2737
2738	/**
2739	* Sets up the virtual-APIC page address for the VMCS.
2740	*
2741	* @returns VBox status code.
2742	* @param pVCpu The cross context virtual CPU structure.
2743	* @param pVmcsInfo The VMCS info. object.
2744	*/
2745	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2746	{
2747	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2748	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
2749	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
2750	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2751	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
2752	}
2753
2754
2755	/**
2756	* Sets up the MSR-bitmap address for the VMCS.
2757	*
2758	* @returns VBox status code.
2759	* @param pVCpu The cross context virtual CPU structure.
2760	* @param pVmcsInfo The VMCS info. object.
2761	*/
2762	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2763	{
2764	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2765	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
2766	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
2767	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2768	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
2769	}
2770
2771
2772	/**
2773	* Sets up the APIC-access page address for the VMCS.
2774	*
2775	* @returns VBox status code.
2776	* @param pVCpu The cross context virtual CPU structure.
2777	*/
2778	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
2779	{
2780	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
2781	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
2782	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2783	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
2784	}
2785
2786
2787	/**
2788	* Sets up the VMCS link pointer for the VMCS.
2789	*
2790	* @returns VBox status code.
2791	* @param pVCpu The cross context virtual CPU structure.
2792	* @param pVmcsInfo The VMCS info. object.
2793	*/
2794	DECLINLINE(int) hmR0VmxSetupVmcsLinkPtr(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2795	{
2796	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2797	uint64_t const u64VmcsLinkPtr = pVmcsInfo->u64VmcsLinkPtr;
2798	Assert(u64VmcsLinkPtr == UINT64_C(0xffffffffffffffff)); /* Bits 63:0 MB1. */
2799	return VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, u64VmcsLinkPtr);
2800	}
2801
2802
2803	/**
2804	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
2805	* in the VMCS.
2806	*
2807	* @returns VBox status code.
2808	* @param pVCpu The cross context virtual CPU structure.
2809	* @param pVmcsInfo The VMCS info. object.
2810	*/
2811	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2812	{
2813	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2814
2815	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
2816	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
2817	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2818
2819	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
2820	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
2821	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
2822
2823	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
2824	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
2825	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2826
2827	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
2828	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
2829	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
2830	AssertRCReturn(rc, rc);
2831	return VINF_SUCCESS;
2832	}
2833
2834
2835	/**
2836	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
2837	*
2838	* @param pVCpu The cross context virtual CPU structure.
2839	* @param pVmcsInfo The VMCS info. object.
2840	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2841	*/
2842	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2843	{
2844	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
2845
2846	/*
2847	* The guest can access the following MSRs (read, write) without causing
2848	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
2849	*/
2850	PVM pVM = pVCpu->CTX_SUFF(pVM);
2851	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
2852	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
2853	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
2854	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2855	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
2856
2857	/*
2858	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
2859	* associated with then. We never need to intercept access (writes need to be
2860	* executed without causing a VM-exit, reads will #GP fault anyway).
2861	*
2862	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
2863	* read/write them. We swap the the guest/host MSR value using the
2864	* auto-load/store MSR area.
2865	*/
2866	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2867	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
2868	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
2869	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
2870	if (pVM->cpum.ro.GuestFeatures.fIbrs)
2871	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
2872
2873	#if HC_ARCH_BITS == 64
2874	/*
2875	* Allow full read/write access for the following MSRs (mandatory for VT-x)
2876	* required for 64-bit guests.
2877	*/
2878	if (pVM->hm.s.fAllow64BitGuests)
2879	{
2880	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
2881	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
2882	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
2883	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2884	}
2885	#endif
2886
2887	/*
2888	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
2889	*/
2890	#ifdef VBOX_STRICT
2891	Assert(pVmcsInfo->pvMsrBitmap);
2892	uint32_t const fMsrpmEfer = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
2893	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
2894	#endif
2895	}
2896
2897
2898	/**
2899	* Sets up pin-based VM-execution controls in the VMCS.
2900	*
2901	* @returns VBox status code.
2902	* @param pVCpu The cross context virtual CPU structure.
2903	* @param pVmcsInfo The VMCS info. object.
2904	*/
2905	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2906	{
2907	PVM pVM = pVCpu->CTX_SUFF(pVM);
2908	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
2909	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2910
2911	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2912	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2913
2914	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2915	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2916
2917	/* Enable the VMX-preemption timer. */
2918	if (pVM->hm.s.vmx.fUsePreemptTimer)
2919	{
2920	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2921	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
2922	}
2923
2924	#if 0
2925	/* Enable posted-interrupt processing. */
2926	if (pVM->hm.s.fPostedIntrs)
2927	{
2928	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2929	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2930	fVal \|= VMX_PIN_CTL_POSTED_INT;
2931	}
2932	#endif
2933
2934	if ((fVal & fZap) != fVal)
2935	{
2936	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2937	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
2938	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2939	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2940	}
2941
2942	/* Commit it to the VMCS and update our cache. */
2943	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2944	AssertRCReturn(rc, rc);
2945	pVmcsInfo->u32PinCtls = fVal;
2946
2947	return VINF_SUCCESS;
2948	}
2949
2950
2951	/**
2952	* Sets up secondary processor-based VM-execution controls in the VMCS.
2953	*
2954	* @returns VBox status code.
2955	* @param pVCpu The cross context virtual CPU structure.
2956	* @param pVmcsInfo The VMCS info. object.
2957	*/
2958	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2959	{
2960	PVM pVM = pVCpu->CTX_SUFF(pVM);
2961	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
2962	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2963
2964	/* WBINVD causes a VM-exit. */
2965	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2966	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
2967
2968	/* Enable EPT (aka nested-paging). */
2969	if (pVM->hm.s.fNestedPaging)
2970	fVal \|= VMX_PROC_CTLS2_EPT;
2971
2972	/* Enable the INVPCID instruction if supported by the hardware and we expose
2973	it to the guest. Without this, guest executing INVPCID would cause a #UD. */
2974	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2975	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
2976	fVal \|= VMX_PROC_CTLS2_INVPCID;
2977
2978	/* Enable VPID. */
2979	if (pVM->hm.s.vmx.fVpid)
2980	fVal \|= VMX_PROC_CTLS2_VPID;
2981
2982	/* Enable unrestricted guest execution. */
2983	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2984	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
2985
2986	#if 0
2987	if (pVM->hm.s.fVirtApicRegs)
2988	{
2989	/* Enable APIC-register virtualization. */
2990	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
2991	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
2992
2993	/* Enable virtual-interrupt delivery. */
2994	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
2995	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
2996	}
2997	#endif
2998
2999	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
3000	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3001	* done dynamically. */
3002	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3003	{
3004	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3005	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3006	AssertRCReturn(rc, rc);
3007	}
3008
3009	/* Enable the RDTSCP instruction if supported by the hardware and we expose
3010	it to the guest. Without this, guest executing RDTSCP would cause a #UD. */
3011	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
3012	&& pVM->cpum.ro.GuestFeatures.fRdTscP)
3013	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3014
3015	/* Enable Pause-Loop exiting. */
3016	if ( pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT
3017	&& pVM->hm.s.vmx.cPleGapTicks
3018	&& pVM->hm.s.vmx.cPleWindowTicks)
3019	{
3020	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3021
3022	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3023	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3024	AssertRCReturn(rc, rc);
3025	}
3026
3027	if ((fVal & fZap) != fVal)
3028	{
3029	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3030	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3031	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3032	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3033	}
3034
3035	/* Commit it to the VMCS and update our cache. */
3036	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3037	AssertRCReturn(rc, rc);
3038	pVmcsInfo->u32ProcCtls2 = fVal;
3039
3040	return VINF_SUCCESS;
3041	}
3042
3043
3044	/**
3045	* Sets up processor-based VM-execution controls in the VMCS.
3046	*
3047	* @returns VBox status code.
3048	* @param pVCpu The cross context virtual CPU structure.
3049	* @param pVmcsInfo The VMCS info. object.
3050	*/
3051	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3052	{
3053	PVM pVM = pVCpu->CTX_SUFF(pVM);
3054
3055	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3056	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3057
3058	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3059	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3060	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3061	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3062	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3063	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3064	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3065
3066	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3067	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3068	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3069	{
3070	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3071	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3072	}
3073
3074	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3075	if (!pVM->hm.s.fNestedPaging)
3076	{
3077	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3078	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3079	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3080	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3081	}
3082
3083	/* Use TPR shadowing if supported by the CPU. */
3084	if ( PDMHasApic(pVM)
3085	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3086	{
3087	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3088	/* CR8 writes cause a VM-exit based on TPR threshold. */
3089	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3090	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3091	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3092	AssertRCReturn(rc, rc);
3093	}
3094	else
3095	{
3096	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3097	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3098	if (pVM->hm.s.fAllow64BitGuests)
3099	{
3100	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3101	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3102	}
3103	}
3104
3105	/* Use MSR-bitmaps if supported by the CPU. */
3106	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3107	{
3108	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3109	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3110	AssertRCReturn(rc, rc);
3111	}
3112
3113	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3114	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3115	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3116
3117	if ((fVal & fZap) != fVal)
3118	{
3119	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3120	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3121	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3122	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3123	}
3124
3125	/* Commit it to the VMCS and update our cache. */
3126	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3127	AssertRCReturn(rc, rc);
3128	pVmcsInfo->u32ProcCtls = fVal;
3129
3130	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3131	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3132	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo, false /* fIsNstGstVmcs */);
3133
3134	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3135	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3136	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3137
3138	/* Sanity check, should not really happen. */
3139	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3140	{ /* likely */ }
3141	else
3142	{
3143	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3144	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3145	}
3146
3147	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3148	return VINF_SUCCESS;
3149	}
3150
3151
3152	/**
3153	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3154	* Processor-based VM-execution) control fields in the VMCS.
3155	*
3156	* @returns VBox status code.
3157	* @param pVCpu The cross context virtual CPU structure.
3158	* @param pVmcsInfo The VMCS info. object.
3159	*/
3160	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3161	{
3162	/* Set the auto-load/store MSR area addresses in the VMCS. */
3163	int rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3164	if (RT_SUCCESS(rc))
3165	{
3166	/* Set the VMCS link pointer in the VMCS. */
3167	rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3168	if (RT_SUCCESS(rc))
3169	{
3170	/* Set the CR0/CR4 guest/host mask. */
3171	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3172	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3173	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3174	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3175	if (RT_SUCCESS(rc))
3176	{
3177	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3178	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3179	return VINF_SUCCESS;
3180	}
3181	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3182	}
3183	else
3184	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3185	}
3186	else
3187	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3188	return rc;
3189	}
3190
3191
3192	/**
3193	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3194	*
3195	* We shall setup those exception intercepts that don't change during the
3196	* lifetime of the VM here. The rest are done dynamically while loading the
3197	* guest state.
3198	*
3199	* @returns VBox status code.
3200	* @param pVCpu The cross context virtual CPU structure.
3201	* @param pVmcsInfo The VMCS info. object.
3202	*/
3203	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3204	{
3205	/*
3206	* The following exceptions are always intercepted:
3207	*
3208	* #AC - To prevent the guest from hanging the CPU.
3209	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3210	* recursive #DBs can cause a CPU hang.
3211	* #PF - To sync our shadow page tables when nested-paging is not used.
3212	*/
3213	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3214	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3215	\| RT_BIT(X86_XCPT_DB)
3216	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3217
3218	/* Commit it to the VMCS. */
3219	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3220	AssertRCReturn(rc, rc);
3221
3222	/* Update our cache of the exception bitmap. */
3223	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3224	return VINF_SUCCESS;
3225	}
3226
3227
3228	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3229	/**
3230	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3231	*
3232	* @returns VBox status code.
3233	* @param pVCpu The cross context virtual CPU structure.
3234	* @param pVmcsInfo The VMCS info. object.
3235	*/
3236	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3237	{
3238	PVM pVM = pVCpu->CTX_SUFF(pVM);
3239	int rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3240	if (RT_SUCCESS(rc))
3241	{
3242	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3243	if (RT_SUCCESS(rc))
3244	{
3245	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3246	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3247	if (RT_SUCCESS(rc))
3248	{
3249	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3250	rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3251	if (RT_SUCCESS(rc))
3252	return VINF_SUCCESS;
3253
3254	LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3255	}
3256	else
3257	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3258	}
3259	else
3260	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3261	}
3262	else
3263	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3264
3265	return rc;
3266	}
3267	#endif
3268
3269
3270	/**
3271	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3272	* VMX.
3273	*
3274	* @returns VBox status code.
3275	* @param pVCpu The cross context virtual CPU structure.
3276	* @param pVmcsInfo The VMCS info. object.
3277	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3278	*/
3279	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3280	{
3281	Assert(pVmcsInfo);
3282	Assert(pVmcsInfo->pvVmcs);
3283	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3284
3285	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3286	PVM pVM = pVCpu->CTX_SUFF(pVM);
3287	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3288	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3289
3290	LogFlowFunc(("\n"));
3291
3292	/*
3293	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3294	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3295	*/
3296	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3297	if (RT_SUCCESS(rc))
3298	{
3299	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3300	if (RT_SUCCESS(rc))
3301	{
3302	if (!fIsNstGstVmcs)
3303	{
3304	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3305	if (RT_SUCCESS(rc))
3306	{
3307	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
3308	if (RT_SUCCESS(rc))
3309	{
3310	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
3311	if (RT_SUCCESS(rc))
3312	{
3313	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
3314	if (RT_SUCCESS(rc))
3315	{ /* likely */ }
3316	else
3317	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
3318	}
3319	else
3320	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
3321	}
3322	else
3323	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
3324	}
3325	else
3326	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
3327	}
3328	else
3329	{
3330	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3331	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
3332	if (RT_SUCCESS(rc))
3333	{ /* likely */ }
3334	else
3335	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
3336	#else
3337	AssertFailed();
3338	#endif
3339	}
3340	}
3341	else
3342	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
3343	}
3344	else
3345	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
3346
3347	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
3348	if (RT_SUCCESS(rc))
3349	{
3350	rc = hmR0VmxClearVmcs(pVmcsInfo);
3351	if (RT_SUCCESS(rc))
3352	{ /* likely */ }
3353	else
3354	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
3355	}
3356
3357	/*
3358	* Update the last-error record both for failures and success, so we
3359	* can propagate the status code back to ring-3 for diagnostics.
3360	*/
3361	hmR0VmxUpdateErrorRecord(pVCpu, rc);
3362	NOREF(pszVmcs);
3363	return rc;
3364	}
3365
3366
3367	/**
3368	* Does global VT-x initialization (called during module initialization).
3369	*
3370	* @returns VBox status code.
3371	*/
3372	VMMR0DECL(int) VMXR0GlobalInit(void)
3373	{
3374	#ifdef HMVMX_USE_FUNCTION_TABLE
3375	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
3376	# ifdef VBOX_STRICT
3377	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
3378	Assert(g_apfnVMExitHandlers[i]);
3379	# endif
3380	#endif
3381	return VINF_SUCCESS;
3382	}
3383
3384
3385	/**
3386	* Does global VT-x termination (called during module termination).
3387	*/
3388	VMMR0DECL(void) VMXR0GlobalTerm()
3389	{
3390	/* Nothing to do currently. */
3391	}
3392
3393
3394	/**
3395	* Sets up and activates VT-x on the current CPU.
3396	*
3397	* @returns VBox status code.
3398	* @param pHostCpu The HM physical-CPU structure.
3399	* @param pVM The cross context VM structure. Can be
3400	* NULL after a host resume operation.
3401	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
3402	* fEnabledByHost is @c true).
3403	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
3404	* @a fEnabledByHost is @c true).
3405	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
3406	* enable VT-x on the host.
3407	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
3408	*/
3409	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
3410	PCSUPHWVIRTMSRS pHwvirtMsrs)
3411	{
3412	Assert(pHostCpu);
3413	Assert(pHwvirtMsrs);
3414	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3415
3416	/* Enable VT-x if it's not already enabled by the host. */
3417	if (!fEnabledByHost)
3418	{
3419	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
3420	if (RT_FAILURE(rc))
3421	return rc;
3422	}
3423
3424	/*
3425	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
3426	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
3427	* invalidated when flushing by VPID.
3428	*/
3429	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3430	{
3431	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
3432	pHostCpu->fFlushAsidBeforeUse = false;
3433	}
3434	else
3435	pHostCpu->fFlushAsidBeforeUse = true;
3436
3437	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
3438	++pHostCpu->cTlbFlushes;
3439
3440	return VINF_SUCCESS;
3441	}
3442
3443
3444	/**
3445	* Deactivates VT-x on the current CPU.
3446	*
3447	* @returns VBox status code.
3448	* @param pvCpuPage Pointer to the VMXON region.
3449	* @param HCPhysCpuPage Physical address of the VMXON region.
3450	*
3451	* @remarks This function should never be called when SUPR0EnableVTx() or
3452	* similar was used to enable VT-x on the host.
3453	*/
3454	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
3455	{
3456	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
3457
3458	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3459	return hmR0VmxLeaveRootMode();
3460	}
3461
3462
3463	/**
3464	* Does per-VM VT-x initialization.
3465	*
3466	* @returns VBox status code.
3467	* @param pVM The cross context VM structure.
3468	*/
3469	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
3470	{
3471	LogFlowFunc(("pVM=%p\n", pVM));
3472
3473	int rc = hmR0VmxStructsAlloc(pVM);
3474	if (RT_FAILURE(rc))
3475	{
3476	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
3477	return rc;
3478	}
3479
3480	return VINF_SUCCESS;
3481	}
3482
3483
3484	/**
3485	* Does per-VM VT-x termination.
3486	*
3487	* @returns VBox status code.
3488	* @param pVM The cross context VM structure.
3489	*/
3490	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
3491	{
3492	LogFlowFunc(("pVM=%p\n", pVM));
3493
3494	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3495	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
3496	{
3497	Assert(pVM->hm.s.vmx.pvScratch);
3498	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
3499	}
3500	#endif
3501	hmR0VmxStructsFree(pVM);
3502	return VINF_SUCCESS;
3503	}
3504
3505
3506	/**
3507	* Sets up the VM for execution using hardware-assisted VMX.
3508	* This function is only called once per-VM during initialization.
3509	*
3510	* @returns VBox status code.
3511	* @param pVM The cross context VM structure.
3512	*/
3513	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
3514	{
3515	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
3516	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3517
3518	LogFlowFunc(("pVM=%p\n", pVM));
3519
3520	/*
3521	* At least verify if VMX is enabled, since we can't check if we're in
3522	* VMX root mode or not without causing a #GP.
3523	*/
3524	RTCCUINTREG const uHostCR4 = ASMGetCR4();
3525	if (RT_LIKELY(uHostCR4 & X86_CR4_VMXE))
3526	{ /* likely */ }
3527	else
3528	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
3529
3530	/*
3531	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
3532	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
3533	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
3534	*/
3535	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3536	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
3537	\|\| !pVM->hm.s.vmx.pRealModeTSS))
3538	{
3539	LogRelFunc(("Invalid real-on-v86 state.\n"));
3540	return VERR_INTERNAL_ERROR;
3541	}
3542
3543	/* Initialize these always, see hmR3InitFinalizeR0().*/
3544	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
3545	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
3546
3547	/* Setup the tagged-TLB flush handlers. */
3548	int rc = hmR0VmxSetupTaggedTlb(pVM);
3549	if (RT_FAILURE(rc))
3550	{
3551	LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
3552	return rc;
3553	}
3554
3555	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
3556	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
3557	#if HC_ARCH_BITS == 64
3558	if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
3559	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
3560	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
3561	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
3562	#endif
3563
3564	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3565	{
3566	PVMCPU pVCpu = &pVM->aCpus[idCpu];
3567	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
3568
3569	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
3570	if (RT_SUCCESS(rc))
3571	{
3572	#if HC_ARCH_BITS == 32
3573	hmR0VmxInitVmcsReadCache(pVCpu);
3574	#endif
3575	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3576	if (pVM->cpum.ro.GuestFeatures.fVmx)
3577	{
3578	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
3579	if (RT_SUCCESS(rc))
3580	{ /* likely */ }
3581	else
3582	{
3583	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
3584	return rc;
3585	}
3586	}
3587	#endif
3588	}
3589	else
3590	{
3591	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
3592	return rc;
3593	}
3594	}
3595
3596	return VINF_SUCCESS;
3597	}
3598
3599
3600	#if HC_ARCH_BITS == 32
3601	# ifdef VBOX_ENABLE_64_BITS_GUESTS
3602	/**
3603	* Check if guest state allows safe use of 32-bit switcher again.
3604	*
3605	* Segment bases and protected mode structures must be 32-bit addressable
3606	* because the 32-bit switcher will ignore high dword when writing these VMCS
3607	* fields. See @bugref{8432} for details.
3608	*
3609	* @returns true if safe, false if must continue to use the 64-bit switcher.
3610	* @param pCtx Pointer to the guest-CPU context.
3611	*
3612	* @remarks No-long-jump zone!!!
3613	*/
3614	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
3615	{
3616	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
3617	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
3618	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3619	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3620	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
3621	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3622	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
3623	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
3624	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3625	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3626
3627	/* All good, bases are 32-bit. */
3628	return true;
3629	}
3630	# endif /* VBOX_ENABLE_64_BITS_GUESTS */
3631
3632	# ifdef VBOX_STRICT
3633	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
3634	{
3635	switch (idxField)
3636	{
3637	case VMX_VMCS_GUEST_RIP:
3638	case VMX_VMCS_GUEST_RSP:
3639	case VMX_VMCS_GUEST_SYSENTER_EIP:
3640	case VMX_VMCS_GUEST_SYSENTER_ESP:
3641	case VMX_VMCS_GUEST_GDTR_BASE:
3642	case VMX_VMCS_GUEST_IDTR_BASE:
3643	case VMX_VMCS_GUEST_CS_BASE:
3644	case VMX_VMCS_GUEST_DS_BASE:
3645	case VMX_VMCS_GUEST_ES_BASE:
3646	case VMX_VMCS_GUEST_FS_BASE:
3647	case VMX_VMCS_GUEST_GS_BASE:
3648	case VMX_VMCS_GUEST_SS_BASE:
3649	case VMX_VMCS_GUEST_LDTR_BASE:
3650	case VMX_VMCS_GUEST_TR_BASE:
3651	case VMX_VMCS_GUEST_CR3:
3652	return true;
3653	}
3654	return false;
3655	}
3656
3657	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3658	{
3659	switch (idxField)
3660	{
3661	/* Read-only fields. */
3662	case VMX_VMCS_RO_EXIT_QUALIFICATION:
3663	return true;
3664	}
3665	/* Remaining readable fields should also be writable. */
3666	return hmR0VmxIsValidWriteField(idxField);
3667	}
3668	# endif /* VBOX_STRICT */
3669
3670
3671	/**
3672	* Executes the specified handler in 64-bit mode.
3673	*
3674	* @returns VBox status code (no informational status codes).
3675	* @param pVCpu The cross context virtual CPU structure.
3676	* @param enmOp The operation to perform.
3677	* @param cParams Number of parameters.
3678	* @param paParam Array of 32-bit parameters.
3679	*/
3680	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
3681	{
3682	PVM pVM = pVCpu->CTX_SUFF(pVM);
3683	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
3684	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
3685	Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
3686	Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
3687
3688	#ifdef VBOX_STRICT
3689	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
3690	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
3691
3692	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
3693	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
3694	#endif
3695
3696	/* Disable interrupts. */
3697	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
3698
3699	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
3700	RTCPUID idHostCpu = RTMpCpuId();
3701	CPUMR0SetLApic(pVCpu, idHostCpu);
3702	#endif
3703
3704	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
3705
3706	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3707	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3708
3709	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
3710	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3711	hmR0VmxClearVmcs(pVmcsInfo);
3712
3713	/* Leave VMX root mode and disable VMX. */
3714	VMXDisable();
3715	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3716
3717	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
3718	CPUMSetHyperEIP(pVCpu, enmOp);
3719	for (int i = (int)cParams - 1; i >= 0; i--)
3720	CPUMPushHyper(pVCpu, paParam[i]);
3721
3722	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
3723
3724	/* Call the switcher. */
3725	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
3726	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
3727
3728	/* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
3729	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
3730
3731	/* Re-enter VMX root mode. */
3732	int rc2 = VMXEnable(HCPhysCpuPage);
3733	if (RT_FAILURE(rc2))
3734	{
3735	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3736	ASMSetFlags(fOldEFlags);
3737	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
3738	return rc2;
3739	}
3740
3741	/* Restore the VMCS as the current VMCS. */
3742	rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
3743	AssertRC(rc2);
3744	Assert(!(ASMGetFlags() & X86_EFL_IF));
3745	ASMSetFlags(fOldEFlags);
3746	return rc;
3747	}
3748
3749
3750	/**
3751	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
3752	* supporting 64-bit guests.
3753	*
3754	* @returns VBox status code.
3755	* @param fResume Whether to VMLAUNCH or VMRESUME.
3756	* @param pCtx Pointer to the guest-CPU context.
3757	* @param pCache Pointer to the VMCS batch cache.
3758	* @param pVM The cross context VM structure.
3759	* @param pVCpu The cross context virtual CPU structure.
3760	*/
3761	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
3762	{
3763	NOREF(fResume);
3764
3765	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3766	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3767	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3768
3769	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3770	pCache->uPos = 1;
3771	pCache->interPD = PGMGetInterPaeCR3(pVM);
3772	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
3773	#endif
3774
3775	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3776	pCache->TestIn.HCPhysCpuPage = 0;
3777	pCache->TestIn.HCPhysVmcs = 0;
3778	pCache->TestIn.pCache = 0;
3779	pCache->TestOut.HCPhysVmcs = 0;
3780	pCache->TestOut.pCache = 0;
3781	pCache->TestOut.pCtx = 0;
3782	pCache->TestOut.eflags = 0;
3783	#else
3784	NOREF(pCache);
3785	#endif
3786
3787	uint32_t aParam[10];
3788	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
3789	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
3790	aParam[2] = RT_LO_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
3791	aParam[3] = RT_HI_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
3792	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
3793	aParam[5] = 0;
3794	aParam[6] = VM_RC_ADDR(pVM, pVM);
3795	aParam[7] = 0;
3796	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
3797	aParam[9] = 0;
3798
3799	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3800	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
3801	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
3802	#endif
3803	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
3804
3805	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3806	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
3807	Assert(pCtx->dr[4] == 10);
3808	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
3809	#endif
3810
3811	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3812	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
3813	AssertMsg(pCache->TestIn.HCPhysVmcs == pVmcsInfo->HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3814	pVmcsInfo->HCPhysVmcs));
3815	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3816	pCache->TestOut.HCPhysVmcs));
3817	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
3818	pCache->TestOut.pCache));
3819	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
3820	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
3821	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
3822	pCache->TestOut.pCtx));
3823	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
3824	#endif
3825	NOREF(pCtx);
3826	return rc;
3827	}
3828	#endif
3829
3830
3831	/**
3832	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
3833	* the VMCS.
3834	*
3835	* @returns VBox status code.
3836	*/
3837	static int hmR0VmxExportHostControlRegs(void)
3838	{
3839	RTCCUINTREG uReg = ASMGetCR0();
3840	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
3841	AssertRCReturn(rc, rc);
3842
3843	uReg = ASMGetCR3();
3844	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
3845	AssertRCReturn(rc, rc);
3846
3847	uReg = ASMGetCR4();
3848	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
3849	AssertRCReturn(rc, rc);
3850	return rc;
3851	}
3852
3853
3854	/**
3855	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
3856	* the host-state area in the VMCS.
3857	*
3858	* @returns VBox status code.
3859	* @param pVCpu The cross context virtual CPU structure.
3860	*/
3861	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
3862	{
3863	#if HC_ARCH_BITS == 64
3864	/**
3865	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
3866	* requirements. See hmR0VmxExportHostSegmentRegs().
3867	*/
3868	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
3869	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
3870	{ \
3871	bool fValidSelector = true; \
3872	if ((selValue) & X86_SEL_LDT) \
3873	{ \
3874	uint32_t uAttr = ASMGetSegAttr((selValue)); \
3875	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
3876	} \
3877	if (fValidSelector) \
3878	{ \
3879	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
3880	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
3881	} \
3882	(selValue) = 0; \
3883	}
3884
3885	/*
3886	* If we've executed guest code using hardware-assisted VMX, the host-state bits
3887	* will be messed up. We should -not- save the messed up state without restoring
3888	* the original host-state, see @bugref{7240}.
3889	*
3890	* This apparently can happen (most likely the FPU changes), deal with it rather than
3891	* asserting. Was observed booting Solaris 10u10 32-bit guest.
3892	*/
3893	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
3894	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
3895	{
3896	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
3897	pVCpu->idCpu));
3898	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
3899	}
3900	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
3901	#else
3902	RT_NOREF(pVCpu);
3903	#endif
3904
3905	/*
3906	* Host DS, ES, FS and GS segment registers.
3907	*/
3908	#if HC_ARCH_BITS == 64
3909	RTSEL uSelDS = ASMGetDS();
3910	RTSEL uSelES = ASMGetES();
3911	RTSEL uSelFS = ASMGetFS();
3912	RTSEL uSelGS = ASMGetGS();
3913	#else
3914	RTSEL uSelDS = 0;
3915	RTSEL uSelES = 0;
3916	RTSEL uSelFS = 0;
3917	RTSEL uSelGS = 0;
3918	#endif
3919
3920	/*
3921	* Host CS and SS segment registers.
3922	*/
3923	RTSEL uSelCS = ASMGetCS();
3924	RTSEL uSelSS = ASMGetSS();
3925
3926	/*
3927	* Host TR segment register.
3928	*/
3929	RTSEL uSelTR = ASMGetTR();
3930
3931	#if HC_ARCH_BITS == 64
3932	/*
3933	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
3934	* gain VM-entry and restore them before we get preempted.
3935	*
3936	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
3937	*/
3938	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3939	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3940	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3941	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3942	# undef VMXLOCAL_ADJUST_HOST_SEG
3943	#endif
3944
3945	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
3946	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
3947	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
3948	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
3949	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
3950	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
3951	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
3952	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
3953	Assert(uSelCS);
3954	Assert(uSelTR);
3955
3956	/* Write these host selector fields into the host-state area in the VMCS. */
3957	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3958	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3959	#if HC_ARCH_BITS == 64
3960	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3961	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3962	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3963	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3964	#else
3965	NOREF(uSelDS);
3966	NOREF(uSelES);
3967	NOREF(uSelFS);
3968	NOREF(uSelGS);
3969	#endif
3970	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3971	AssertRCReturn(rc, rc);
3972
3973	/*
3974	* Host GDTR and IDTR.
3975	*/
3976	RTGDTR Gdtr;
3977	RTIDTR Idtr;
3978	RT_ZERO(Gdtr);
3979	RT_ZERO(Idtr);
3980	ASMGetGDTR(&Gdtr);
3981	ASMGetIDTR(&Idtr);
3982	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3983	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3984	AssertRCReturn(rc, rc);
3985
3986	#if HC_ARCH_BITS == 64
3987	/*
3988	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
3989	* them to the maximum limit (0xffff) on every VM-exit.
3990	*/
3991	if (Gdtr.cbGdt != 0xffff)
3992	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
3993
3994	/*
3995	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
3996	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
3997	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
3998	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
3999	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4000	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4001	* at 0xffff on hosts where we are sure it won't cause trouble.
4002	*/
4003	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4004	if (Idtr.cbIdt < 0x0fff)
4005	# else
4006	if (Idtr.cbIdt != 0xffff)
4007	# endif
4008	{
4009	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4010	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4011	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4012	}
4013	#endif
4014
4015	/*
4016	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4017	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4018	* RPL should be too in most cases.
4019	*/
4020	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4021	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4022
4023	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4024	#if HC_ARCH_BITS == 64
4025	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4026
4027	/*
4028	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4029	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4030	* restoration if the host has something else. Task switching is not supported in 64-bit
4031	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4032	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4033	*
4034	* [1] See Intel spec. 3.5 "System Descriptor Types".
4035	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4036	*/
4037	PVM pVM = pVCpu->CTX_SUFF(pVM);
4038	Assert(pDesc->System.u4Type == 11);
4039	if ( pDesc->System.u16LimitLow != 0x67
4040	\|\| pDesc->System.u4LimitHigh)
4041	{
4042	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4043	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4044	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4045	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4046	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4047	}
4048
4049	/*
4050	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4051	*/
4052	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4053	{
4054	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4055	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4056	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4057	{
4058	/* The GDT is read-only but the writable GDT is available. */
4059	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4060	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4061	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4062	AssertRCReturn(rc, rc);
4063	}
4064	}
4065	#else
4066	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4067	#endif
4068	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4069	AssertRCReturn(rc, rc);
4070
4071	/*
4072	* Host FS base and GS base.
4073	*/
4074	#if HC_ARCH_BITS == 64
4075	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4076	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4077	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4078	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4079	AssertRCReturn(rc, rc);
4080
4081	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4082	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4083	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4084	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4085	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4086	#endif
4087	return VINF_SUCCESS;
4088	}
4089
4090
4091	/**
4092	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4093	* host-state area of the VMCS.
4094	*
4095	* These MSRs will be automatically restored on the host after every successful
4096	* VM-exit.
4097	*
4098	* @returns VBox status code.
4099	* @param pVCpu The cross context virtual CPU structure.
4100	*
4101	* @remarks No-long-jump zone!!!
4102	*/
4103	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4104	{
4105	AssertPtr(pVCpu);
4106
4107	/*
4108	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4109	* rather than swapping them on every VM-entry.
4110	*/
4111	hmR0VmxLazySaveHostMsrs(pVCpu);
4112
4113	/*
4114	* Host Sysenter MSRs.
4115	*/
4116	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4117	#if HC_ARCH_BITS == 32
4118	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4119	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4120	#else
4121	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4122	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4123	#endif
4124	AssertRCReturn(rc, rc);
4125
4126	/*
4127	* Host EFER MSR.
4128	*
4129	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4130	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4131	*/
4132	PVM pVM = pVCpu->CTX_SUFF(pVM);
4133	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4134	{
4135	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4136	AssertRCReturn(rc, rc);
4137	}
4138
4139	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4140	* hmR0VmxExportGuestEntryExitCtls(). */
4141
4142	return VINF_SUCCESS;
4143	}
4144
4145
4146	/**
4147	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4148	*
4149	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4150	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4151	*
4152	* @returns true if we need to load guest EFER, false otherwise.
4153	* @param pVCpu The cross context virtual CPU structure.
4154	*
4155	* @remarks Requires EFER, CR4.
4156	* @remarks No-long-jump zone!!!
4157	*/
4158	static bool hmR0VmxShouldSwapEferMsr(PCVMCPU pVCpu)
4159	{
4160	#ifdef HMVMX_ALWAYS_SWAP_EFER
4161	RT_NOREF(pVCpu);
4162	return true;
4163	#else
4164	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4165	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4166	/* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4167	if (CPUMIsGuestInLongModeEx(pCtx))
4168	return false;
4169	#endif
4170
4171	PVM pVM = pVCpu->CTX_SUFF(pVM);
4172	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4173	uint64_t const u64GuestEfer = pCtx->msrEFER;
4174
4175	/*
4176	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4177	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4178	*/
4179	if ( CPUMIsGuestInLongModeEx(pCtx)
4180	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4181	return true;
4182
4183	/*
4184	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4185	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4186	*
4187	* See Intel spec. 4.5 "IA-32e Paging".
4188	* See Intel spec. 4.1.1 "Three Paging Modes".
4189	*
4190	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4191	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4192	*/
4193	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4194	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4195	if ( (pCtx->cr4 & X86_CR4_PAE)
4196	&& (pCtx->cr0 & X86_CR0_PG)
4197	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4198	{
4199	/* Assert that host is NX capable. */
4200	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4201	return true;
4202	}
4203
4204	return false;
4205	#endif
4206	}
4207
4208	/**
4209	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4210	* VMCS.
4211	*
4212	* This is typically required when the guest changes paging mode.
4213	*
4214	* @returns VBox status code.
4215	* @param pVCpu The cross context virtual CPU structure.
4216	* @param pVmxTransient The VMX-transient structure.
4217	*
4218	* @remarks Requires EFER.
4219	* @remarks No-long-jump zone!!!
4220	*/
4221	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4222	{
4223	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4224	{
4225	PVM pVM = pVCpu->CTX_SUFF(pVM);
4226	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4227
4228	/*
4229	* VM-entry controls.
4230	*/
4231	{
4232	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4233	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4234
4235	/*
4236	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4237	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4238	*
4239	* For nested-guests, this is a mandatory VM-entry control. It's also
4240	* required because we do not want to leak host bits to the nested-guest.
4241	*/
4242	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4243
4244	/*
4245	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4246	*
4247	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4248	* required to get the nested-guest working with hardware-assisted VMX execution.
4249	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested-hypervisor
4250	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4251	* here rather than while merging the guest VMCS controls.
4252	*/
4253	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4254	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4255	else
4256	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4257
4258	/*
4259	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4260	*
4261	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4262	* regardless of whether the nested-guest VMCS specifies it because we are free to
4263	* load whatever MSRs we require and we do not need to modify the guest visible copy
4264	* of the VM-entry MSR load area.
4265	*/
4266	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4267	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4268	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4269	else
4270	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4271
4272	/*
4273	* The following should -not- be set (since we're not in SMM mode):
4274	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4275	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4276	*/
4277
4278	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4279	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4280
4281	if ((fVal & fZap) == fVal)
4282	{ /* likely */ }
4283	else
4284	{
4285	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4286	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4287	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4288	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4289	}
4290
4291	/* Commit it to the VMCS. */
4292	if (pVmcsInfo->u32EntryCtls != fVal)
4293	{
4294	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4295	AssertRCReturn(rc, rc);
4296	pVmcsInfo->u32EntryCtls = fVal;
4297	}
4298	}
4299
4300	/*
4301	* VM-exit controls.
4302	*/
4303	{
4304	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4305	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4306
4307	/*
4308	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4309	* supported the 1-setting of this bit.
4310	*
4311	* For nested-guests, we set the "save debug controls" as the converse
4312	* "load debug controls" is mandatory for nested-guests anyway.
4313	*/
4314	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4315
4316	/*
4317	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4318	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4319	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4320	* hmR0VmxExportHostMsrs().
4321	*
4322	* For nested-guests, we always set this bit as we do not support 32-bit
4323	* hosts.
4324	*/
4325	#if HC_ARCH_BITS == 64
4326	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4327	#else
4328	Assert(!pVmxTransient->fIsNestedGuest);
4329	Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
4330	\|\| pVmcsInfo->pfnStartVM == VMXR0StartVM32);
4331	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
4332	if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
4333	{
4334	/* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
4335	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4336	}
4337	else
4338	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
4339	#endif
4340
4341	/*
4342	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4343	*
4344	* For nested-guests, we should use the "save IA32_EFER" control if we also
4345	* used the "load IA32_EFER" control while exporting VM-entry controls.
4346	*/
4347	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4348	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4349	{
4350	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4351	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4352	}
4353
4354	/*
4355	* Enable saving of the VMX-preemption timer value on VM-exit.
4356	* For nested-guests, currently not exposed/used.
4357	*/
4358	if ( pVM->hm.s.vmx.fUsePreemptTimer
4359	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4360	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4361
4362	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4363	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4364
4365	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4366	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4367	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4368
4369	if ((fVal & fZap) == fVal)
4370	{ /* likely */ }
4371	else
4372	{
4373	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4374	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4375	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4376	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4377	}
4378
4379	/* Commit it to the VMCS. */
4380	if (pVmcsInfo->u32ExitCtls != fVal)
4381	{
4382	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4383	AssertRCReturn(rc, rc);
4384	pVmcsInfo->u32ExitCtls = fVal;
4385	}
4386	}
4387
4388	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4389	}
4390	return VINF_SUCCESS;
4391	}
4392
4393
4394	/**
4395	* Sets the TPR threshold in the VMCS.
4396	*
4397	* @returns VBox status code.
4398	* @param pVCpu The cross context virtual CPU structure.
4399	* @param pVmcsInfo The VMCS info. object.
4400	* @param u32TprThreshold The TPR threshold (task-priority class only).
4401	*/
4402	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4403	{
4404	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4405	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4406	RT_NOREF2(pVCpu, pVmcsInfo);
4407	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4408	}
4409
4410
4411	/**
4412	* Exports the guest APIC TPR state into the VMCS.
4413	*
4414	* @returns VBox status code.
4415	* @param pVCpu The cross context virtual CPU structure.
4416	* @param pVmxTransient The VMX-transient structure.
4417	*
4418	* @remarks No-long-jump zone!!!
4419	*/
4420	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4421	{
4422	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4423	{
4424	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4425
4426	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4427	if (!pVmxTransient->fIsNestedGuest)
4428	{
4429	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4430	&& APICIsEnabled(pVCpu))
4431	{
4432	/*
4433	* Setup TPR shadowing.
4434	*/
4435	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4436	{
4437	bool fPendingIntr = false;
4438	uint8_t u8Tpr = 0;
4439	uint8_t u8PendingIntr = 0;
4440	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4441	AssertRCReturn(rc, rc);
4442
4443	/*
4444	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4445	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4446	* priority of the pending interrupt so we can deliver the interrupt. If there
4447	* are no interrupts pending, set threshold to 0 to not cause any
4448	* TPR-below-threshold VM-exits.
4449	*/
4450	Assert(pVmcsInfo->pbVirtApic);
4451	pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
4452	uint32_t u32TprThreshold = 0;
4453	if (fPendingIntr)
4454	{
4455	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4456	(which is the Task-Priority Class). */
4457	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4458	const uint8_t u8TprPriority = u8Tpr >> 4;
4459	if (u8PendingPriority <= u8TprPriority)
4460	u32TprThreshold = u8PendingPriority;
4461	}
4462
4463	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
4464	AssertRCReturn(rc, rc);
4465	}
4466	}
4467	}
4468	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4469	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4470	}
4471	return VINF_SUCCESS;
4472	}
4473
4474
4475	/**
4476	* Gets the guest interruptibility-state.
4477	*
4478	* @returns Guest's interruptibility-state.
4479	* @param pVCpu The cross context virtual CPU structure.
4480	* @param pVmcsInfo The VMCS info. object.
4481	*
4482	* @remarks No-long-jump zone!!!
4483	*/
4484	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
4485	{
4486	/*
4487	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4488	*/
4489	uint32_t fIntrState = 0;
4490	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4491	{
4492	/* If inhibition is active, RIP and RFLAGS should've been updated
4493	(i.e. read previously from the VMCS or from ring-3). */
4494	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4495	#ifdef VBOX_STRICT
4496	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
4497	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4498	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
4499	#endif
4500	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
4501	{
4502	if (pCtx->eflags.Bits.u1IF)
4503	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4504	else
4505	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
4506	}
4507	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4508	{
4509	/*
4510	* We can clear the inhibit force flag as even if we go back to the recompiler
4511	* without executing guest code in VT-x, the flag's condition to be cleared is
4512	* met and thus the cleared state is correct.
4513	*/
4514	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4515	}
4516	}
4517
4518	/*
4519	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
4520	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
4521	* setting this would block host-NMIs and IRET will not clear the blocking.
4522	*
4523	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
4524	*
4525	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
4526	*/
4527	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
4528	&& CPUMIsGuestNmiBlocking(pVCpu))
4529	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
4530
4531	return fIntrState;
4532	}
4533
4534
4535	/**
4536	* Exports the exception intercepts required for guest execution in the VMCS.
4537	*
4538	* @returns VBox status code.
4539	* @param pVCpu The cross context virtual CPU structure.
4540	* @param pVmxTransient The VMX-transient structure.
4541	*
4542	* @remarks No-long-jump zone!!!
4543	*/
4544	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4545	{
4546	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
4547	{
4548	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
4549	if ( !pVmxTransient->fIsNestedGuest
4550	&& pVCpu->hm.s.fGIMTrapXcptUD)
4551	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
4552	else
4553	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
4554
4555	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
4556	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
4557	}
4558	return VINF_SUCCESS;
4559	}
4560
4561
4562	/**
4563	* Exports the guest's RIP into the guest-state area in the VMCS.
4564	*
4565	* @returns VBox status code.
4566	* @param pVCpu The cross context virtual CPU structure.
4567	*
4568	* @remarks No-long-jump zone!!!
4569	*/
4570	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
4571	{
4572	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4573	{
4574	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
4575
4576	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
4577	AssertRCReturn(rc, rc);
4578
4579	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
4580	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
4581	}
4582	return VINF_SUCCESS;
4583	}
4584
4585
4586	/**
4587	* Exports the guest's RSP into the guest-state area in the VMCS.
4588	*
4589	* @returns VBox status code.
4590	* @param pVCpu The cross context virtual CPU structure.
4591	*
4592	* @remarks No-long-jump zone!!!
4593	*/
4594	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
4595	{
4596	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
4597	{
4598	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
4599
4600	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
4601	AssertRCReturn(rc, rc);
4602
4603	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
4604	}
4605	return VINF_SUCCESS;
4606	}
4607
4608
4609	/**
4610	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
4611	*
4612	* @returns VBox status code.
4613	* @param pVCpu The cross context virtual CPU structure.
4614	* @param pVmxTransient The VMX-transient structure.
4615	*
4616	* @remarks No-long-jump zone!!!
4617	*/
4618	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4619	{
4620	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
4621	{
4622	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
4623
4624	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
4625	Let us assert it as such and use 32-bit VMWRITE. */
4626	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
4627	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
4628	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
4629	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
4630
4631	/*
4632	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
4633	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
4634	* can run the real-mode guest code under Virtual 8086 mode.
4635	*/
4636	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4637	if (pVmcsInfo->RealMode.fRealOnV86Active)
4638	{
4639	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4640	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4641	Assert(!pVmxTransient->fIsNestedGuest);
4642	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
4643	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
4644	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
4645	}
4646
4647	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
4648	AssertRCReturn(rc, rc);
4649
4650	/*
4651	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
4652	*
4653	* We must avoid setting any automatic debug exceptions delivery when single-stepping
4654	* through the hypervisor debugger using EFLAGS.TF.
4655	*/
4656	if ( !pVmxTransient->fIsNestedGuest
4657	&& !pVCpu->hm.s.fSingleInstruction
4658	&& fEFlags.Bits.u1TF)
4659	{
4660	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
4661	* premature trips to ring-3 esp since IEM does not yet handle it. */
4662	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
4663	AssertRCReturn(rc, rc);
4664	}
4665	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
4666	* nested-guest VMCS. */
4667
4668	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
4669	Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
4670	}
4671	return VINF_SUCCESS;
4672	}
4673
4674
4675	/**
4676	* Exports the guest CR0 control register into the guest-state area in the VMCS.
4677	*
4678	* The guest FPU state is always pre-loaded hence we don't need to bother about
4679	* sharing FPU related CR0 bits between the guest and host.
4680	*
4681	* @returns VBox status code.
4682	* @param pVCpu The cross context virtual CPU structure.
4683	* @param pVmxTransient The VMX-transient structure.
4684	*
4685	* @remarks No-long-jump zone!!!
4686	*/
4687	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4688	{
4689	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
4690	{
4691	PVM pVM = pVCpu->CTX_SUFF(pVM);
4692	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4693
4694	/*
4695	* Figure out fixed CR0 bits in VMX operation.
4696	*/
4697	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4698	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4699	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4700	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
4701	else
4702	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
4703
4704	if (!pVmxTransient->fIsNestedGuest)
4705	{
4706	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4707	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4708	uint64_t const u64ShadowCr0 = u64GuestCr0;
4709	Assert(!RT_HI_U32(u64GuestCr0));
4710
4711	/*
4712	* Setup VT-x's view of the guest CR0.
4713	*/
4714	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
4715	if (pVM->hm.s.fNestedPaging)
4716	{
4717	if (CPUMIsGuestPagingEnabled(pVCpu))
4718	{
4719	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
4720	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
4721	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
4722	}
4723	else
4724	{
4725	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
4726	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
4727	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
4728	}
4729
4730	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
4731	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4732	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
4733	}
4734	else
4735	{
4736	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
4737	u64GuestCr0 \|= X86_CR0_WP;
4738	}
4739
4740	/*
4741	* Guest FPU bits.
4742	*
4743	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
4744	* using CR0.TS.
4745	*
4746	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
4747	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
4748	*/
4749	u64GuestCr0 \|= X86_CR0_NE;
4750
4751	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
4752	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
4753
4754	/*
4755	* Update exception intercepts.
4756	*/
4757	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
4758	if (pVmcsInfo->RealMode.fRealOnV86Active)
4759	{
4760	Assert(PDMVmmDevHeapIsEnabled(pVM));
4761	Assert(pVM->hm.s.vmx.pRealModeTSS);
4762	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
4763	}
4764	else
4765	{
4766	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
4767	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
4768	if (fInterceptMF)
4769	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
4770	}
4771
4772	/* Additional intercepts for debugging, define these yourself explicitly. */
4773	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4774	uXcptBitmap \|= 0
4775	\| RT_BIT(X86_XCPT_BP)
4776	\| RT_BIT(X86_XCPT_DE)
4777	\| RT_BIT(X86_XCPT_NM)
4778	\| RT_BIT(X86_XCPT_TS)
4779	\| RT_BIT(X86_XCPT_UD)
4780	\| RT_BIT(X86_XCPT_NP)
4781	\| RT_BIT(X86_XCPT_SS)
4782	\| RT_BIT(X86_XCPT_GP)
4783	\| RT_BIT(X86_XCPT_PF)
4784	\| RT_BIT(X86_XCPT_MF)
4785	;
4786	#elif defined(HMVMX_ALWAYS_TRAP_PF)
4787	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
4788	#endif
4789	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
4790	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
4791	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
4792
4793	/* Apply the fixed CR0 bits and enable caching. */
4794	u64GuestCr0 \|= fSetCr0;
4795	u64GuestCr0 &= fZapCr0;
4796	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4797
4798	/* Commit the CR0 and related fields to the guest VMCS. */
4799	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo Fix to 64-bit when we drop 32-bit. */
4800	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4801	if (uProcCtls != pVmcsInfo->u32ProcCtls)
4802	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4803	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
4804	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
4805	AssertRCReturn(rc, rc);
4806
4807	/* Update our caches. */
4808	pVmcsInfo->u32ProcCtls = uProcCtls;
4809	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
4810
4811	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4812	}
4813	else
4814	{
4815	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
4816	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4817	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4818	uint64_t const u64ShadowCr0 = pVmcsNstGst->u64Cr0ReadShadow.u;
4819	Assert(!RT_HI_U32(u64GuestCr0));
4820	Assert(u64GuestCr0 & X86_CR0_NE);
4821
4822	/* Apply the fixed CR0 bits and enable caching. */
4823	u64GuestCr0 \|= fSetCr0;
4824	u64GuestCr0 &= fZapCr0;
4825	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4826
4827	/* Commit the CR0 and CR0 read shadow to the nested-guest VMCS. */
4828	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo NSTVMX: Fix to 64-bit when we drop 32-bit. */
4829	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4830	AssertRCReturn(rc, rc);
4831
4832	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4833	}
4834
4835	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
4836	}
4837
4838	return VINF_SUCCESS;
4839	}
4840
4841
4842	/**
4843	* Exports the guest control registers (CR3, CR4) into the guest-state area
4844	* in the VMCS.
4845	*
4846	* @returns VBox strict status code.
4847	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
4848	* without unrestricted guest access and the VMMDev is not presently
4849	* mapped (e.g. EFI32).
4850	*
4851	* @param pVCpu The cross context virtual CPU structure.
4852	* @param pVmxTransient The VMX-transient structure.
4853	*
4854	* @remarks No-long-jump zone!!!
4855	*/
4856	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4857	{
4858	int rc = VINF_SUCCESS;
4859	PVM pVM = pVCpu->CTX_SUFF(pVM);
4860
4861	/*
4862	* Guest CR2.
4863	* It's always loaded in the assembler code. Nothing to do here.
4864	*/
4865
4866	/*
4867	* Guest CR3.
4868	*/
4869	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
4870	{
4871	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
4872
4873	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
4874	if (pVM->hm.s.fNestedPaging)
4875	{
4876	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4877	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
4878
4879	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
4880	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
4881	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
4882	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
4883
4884	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
4885	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
4886	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
4887
4888	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
4889	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
4890	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
4891	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
4892	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
4893	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
4894	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
4895
4896	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
4897	AssertRCReturn(rc, rc);
4898
4899	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4900	if ( pVM->hm.s.vmx.fUnrestrictedGuest
4901	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
4902	{
4903	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
4904	if (CPUMIsGuestInPAEModeEx(pCtx))
4905	{
4906	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
4907	AssertRCReturn(rc, rc);
4908	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
4909	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
4910	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
4911	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
4912	AssertRCReturn(rc, rc);
4913	}
4914
4915	/*
4916	* The guest's view of its CR3 is unblemished with nested paging when the
4917	* guest is using paging or we have unrestricted guest execution to handle
4918	* the guest when it's not using paging.
4919	*/
4920	GCPhysGuestCR3 = pCtx->cr3;
4921	}
4922	else
4923	{
4924	/*
4925	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
4926	* thinks it accesses physical memory directly, we use our identity-mapped
4927	* page table to map guest-linear to guest-physical addresses. EPT takes care
4928	* of translating it to host-physical addresses.
4929	*/
4930	RTGCPHYS GCPhys;
4931	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4932
4933	/* We obtain it here every time as the guest could have relocated this PCI region. */
4934	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4935	if (RT_SUCCESS(rc))
4936	{ /* likely */ }
4937	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4938	{
4939	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
4940	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4941	}
4942	else
4943	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4944
4945	GCPhysGuestCR3 = GCPhys;
4946	}
4947
4948	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
4949	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4950	AssertRCReturn(rc, rc);
4951	}
4952	else
4953	{
4954	/* Non-nested paging case, just use the hypervisor's CR3. */
4955	RTHCPHYS const HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4956
4957	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
4958	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4959	AssertRCReturn(rc, rc);
4960	}
4961
4962	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
4963	}
4964
4965	/*
4966	* Guest CR4.
4967	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4968	*/
4969	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
4970	{
4971	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4972	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4973	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
4974
4975	/*
4976	* Figure out fixed CR4 bits in VMX operation.
4977	*/
4978	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
4979	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
4980
4981	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
4982	uint64_t u64GuestCr4 = pCtx->cr4;
4983	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest ? pCtx->cr4 : pVmcsNstGst->u64Cr4ReadShadow.u;
4984	Assert(!RT_HI_U32(u64GuestCr4));
4985
4986	/*
4987	* Setup VT-x's view of the guest CR4.
4988	*
4989	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
4990	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
4991	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4992	*
4993	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4994	*/
4995	if (pVmcsInfo->RealMode.fRealOnV86Active)
4996	{
4997	Assert(pVM->hm.s.vmx.pRealModeTSS);
4998	Assert(PDMVmmDevHeapIsEnabled(pVM));
4999	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5000	}
5001
5002	if (pVM->hm.s.fNestedPaging)
5003	{
5004	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5005	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5006	{
5007	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5008	u64GuestCr4 \|= X86_CR4_PSE;
5009	/* Our identity mapping is a 32-bit page directory. */
5010	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5011	}
5012	/* else use guest CR4.*/
5013	}
5014	else
5015	{
5016	Assert(!pVmxTransient->fIsNestedGuest);
5017
5018	/*
5019	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5020	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5021	*/
5022	switch (pVCpu->hm.s.enmShadowMode)
5023	{
5024	case PGMMODE_REAL: /* Real-mode. */
5025	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5026	case PGMMODE_32_BIT: /* 32-bit paging. */
5027	{
5028	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5029	break;
5030	}
5031
5032	case PGMMODE_PAE: /* PAE paging. */
5033	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5034	{
5035	u64GuestCr4 \|= X86_CR4_PAE;
5036	break;
5037	}
5038
5039	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5040	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5041	#ifdef VBOX_ENABLE_64_BITS_GUESTS
5042	break;
5043	#endif
5044	default:
5045	AssertFailed();
5046	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5047	}
5048	}
5049
5050	/* Apply the fixed CR4 bits (mainly CR4.VMXE). */
5051	u64GuestCr4 \|= fSetCr4;
5052	u64GuestCr4 &= fZapCr4;
5053
5054	/* Commit the CR4 and CR4 read shadow to the guest VMCS. */
5055	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u64GuestCr4); /** @todo Fix to 64-bit when we drop 32-bit. */
5056	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
5057	AssertRCReturn(rc, rc);
5058
5059	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5060	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5061
5062	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5063
5064	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5065	}
5066	return rc;
5067	}
5068
5069
5070	/**
5071	* Exports the guest debug registers into the guest-state area in the VMCS.
5072	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5073	*
5074	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5075	*
5076	* @returns VBox status code.
5077	* @param pVCpu The cross context virtual CPU structure.
5078	* @param pVmxTransient The VMX-transient structure.
5079	*
5080	* @remarks No-long-jump zone!!!
5081	*/
5082	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5083	{
5084	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5085
5086	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5087	* stepping. */
5088	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5089	if (pVmxTransient->fIsNestedGuest)
5090	{
5091	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5092	AssertRCReturn(rc, rc);
5093	return VINF_SUCCESS;
5094	}
5095
5096	#ifdef VBOX_STRICT
5097	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5098	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5099	{
5100	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5101	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5102	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5103	}
5104	#endif
5105
5106	bool fSteppingDB = false;
5107	bool fInterceptMovDRx = false;
5108	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5109	if (pVCpu->hm.s.fSingleInstruction)
5110	{
5111	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5112	PVM pVM = pVCpu->CTX_SUFF(pVM);
5113	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5114	{
5115	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5116	Assert(fSteppingDB == false);
5117	}
5118	else
5119	{
5120	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5121	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5122	pVCpu->hm.s.fClearTrapFlag = true;
5123	fSteppingDB = true;
5124	}
5125	}
5126
5127	uint32_t u32GuestDr7;
5128	if ( fSteppingDB
5129	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5130	{
5131	/*
5132	* Use the combined guest and host DRx values found in the hypervisor register set
5133	* because the hypervisor debugger has breakpoints active or someone is single stepping
5134	* on the host side without a monitor trap flag.
5135	*
5136	* Note! DBGF expects a clean DR6 state before executing guest code.
5137	*/
5138	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5139	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5140	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
5141	{
5142	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5143	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
5144	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
5145	}
5146	else
5147	#endif
5148	if (!CPUMIsHyperDebugStateActive(pVCpu))
5149	{
5150	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5151	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5152	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5153	}
5154
5155	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5156	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
5157	pVCpu->hm.s.fUsingHyperDR7 = true;
5158	fInterceptMovDRx = true;
5159	}
5160	else
5161	{
5162	/*
5163	* If the guest has enabled debug registers, we need to load them prior to
5164	* executing guest code so they'll trigger at the right time.
5165	*/
5166	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5167	{
5168	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5169	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5170	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
5171	{
5172	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5173	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
5174	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
5175	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5176	}
5177	else
5178	#endif
5179	if (!CPUMIsGuestDebugStateActive(pVCpu))
5180	{
5181	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5182	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5183	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5184	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5185	}
5186	Assert(!fInterceptMovDRx);
5187	}
5188	/*
5189	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5190	* must intercept #DB in order to maintain a correct DR6 guest value, and
5191	* because we need to intercept it to prevent nested #DBs from hanging the
5192	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5193	*/
5194	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5195	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
5196	&& !CPUMIsGuestDebugStateActive(pVCpu))
5197	#else
5198	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5199	#endif
5200	{
5201	fInterceptMovDRx = true;
5202	}
5203
5204	/* Update DR7 with the actual guest value. */
5205	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5206	pVCpu->hm.s.fUsingHyperDR7 = false;
5207	}
5208
5209	if (fInterceptMovDRx)
5210	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5211	else
5212	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5213
5214	/*
5215	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5216	* monitor-trap flag and update our cache.
5217	*/
5218	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5219	{
5220	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5221	AssertRCReturn(rc2, rc2);
5222	pVmcsInfo->u32ProcCtls = uProcCtls;
5223	}
5224
5225	/*
5226	* Update guest DR7.
5227	*/
5228	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
5229	AssertRCReturn(rc, rc);
5230
5231	/*
5232	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
5233	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
5234	*
5235	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5236	*/
5237	if (fSteppingDB)
5238	{
5239	Assert(pVCpu->hm.s.fSingleInstruction);
5240	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
5241
5242	uint32_t fIntrState = 0;
5243	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
5244	AssertRCReturn(rc, rc);
5245
5246	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5247	{
5248	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5249	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5250	AssertRCReturn(rc, rc);
5251	}
5252	}
5253
5254	return VINF_SUCCESS;
5255	}
5256
5257
5258	#ifdef VBOX_STRICT
5259	/**
5260	* Strict function to validate segment registers.
5261	*
5262	* @param pVCpu The cross context virtual CPU structure.
5263	* @param pVmcsInfo The VMCS info. object.
5264	*
5265	* @remarks Will import guest CR0 on strict builds during validation of
5266	* segments.
5267	*/
5268	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
5269	{
5270	/*
5271	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5272	*
5273	* The reason we check for attribute value 0 in this function and not just the unusable bit is
5274	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
5275	* unusable bit and doesn't change the guest-context value.
5276	*/
5277	PVM pVM = pVCpu->CTX_SUFF(pVM);
5278	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5279	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
5280	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
5281	&& ( !CPUMIsGuestInRealModeEx(pCtx)
5282	&& !CPUMIsGuestInV86ModeEx(pCtx)))
5283	{
5284	/* Protected mode checks */
5285	/* CS */
5286	Assert(pCtx->cs.Attr.n.u1Present);
5287	Assert(!(pCtx->cs.Attr.u & 0xf00));
5288	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
5289	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5290	\|\| !(pCtx->cs.Attr.n.u1Granularity));
5291	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
5292	\|\| (pCtx->cs.Attr.n.u1Granularity));
5293	/* CS cannot be loaded with NULL in protected mode. */
5294	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
5295	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5296	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
5297	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5298	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
5299	else
5300	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
5301	/* SS */
5302	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5303	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
5304	if ( !(pCtx->cr0 & X86_CR0_PE)
5305	\|\| pCtx->cs.Attr.n.u4Type == 3)
5306	{
5307	Assert(!pCtx->ss.Attr.n.u2Dpl);
5308	}
5309	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
5310	{
5311	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5312	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
5313	Assert(pCtx->ss.Attr.n.u1Present);
5314	Assert(!(pCtx->ss.Attr.u & 0xf00));
5315	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
5316	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
5317	\|\| !(pCtx->ss.Attr.n.u1Granularity));
5318	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
5319	\|\| (pCtx->ss.Attr.n.u1Granularity));
5320	}
5321	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
5322	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
5323	{
5324	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5325	Assert(pCtx->ds.Attr.n.u1Present);
5326	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
5327	Assert(!(pCtx->ds.Attr.u & 0xf00));
5328	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
5329	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
5330	\|\| !(pCtx->ds.Attr.n.u1Granularity));
5331	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
5332	\|\| (pCtx->ds.Attr.n.u1Granularity));
5333	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5334	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
5335	}
5336	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
5337	{
5338	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5339	Assert(pCtx->es.Attr.n.u1Present);
5340	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
5341	Assert(!(pCtx->es.Attr.u & 0xf00));
5342	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
5343	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
5344	\|\| !(pCtx->es.Attr.n.u1Granularity));
5345	Assert( !(pCtx->es.u32Limit & 0xfff00000)
5346	\|\| (pCtx->es.Attr.n.u1Granularity));
5347	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5348	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
5349	}
5350	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
5351	{
5352	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5353	Assert(pCtx->fs.Attr.n.u1Present);
5354	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
5355	Assert(!(pCtx->fs.Attr.u & 0xf00));
5356	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
5357	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
5358	\|\| !(pCtx->fs.Attr.n.u1Granularity));
5359	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
5360	\|\| (pCtx->fs.Attr.n.u1Granularity));
5361	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5362	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5363	}
5364	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
5365	{
5366	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5367	Assert(pCtx->gs.Attr.n.u1Present);
5368	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
5369	Assert(!(pCtx->gs.Attr.u & 0xf00));
5370	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
5371	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
5372	\|\| !(pCtx->gs.Attr.n.u1Granularity));
5373	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
5374	\|\| (pCtx->gs.Attr.n.u1Granularity));
5375	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5376	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5377	}
5378	/* 64-bit capable CPUs. */
5379	# if HC_ARCH_BITS == 64
5380	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5381	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
5382	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
5383	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
5384	# endif
5385	}
5386	else if ( CPUMIsGuestInV86ModeEx(pCtx)
5387	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
5388	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
5389	{
5390	/* Real and v86 mode checks. */
5391	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
5392	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
5393	if (pVmcsInfo->RealMode.fRealOnV86Active)
5394	{
5395	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
5396	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
5397	}
5398	else
5399	{
5400	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
5401	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
5402	}
5403
5404	/* CS */
5405	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
5406	Assert(pCtx->cs.u32Limit == 0xffff);
5407	Assert(u32CSAttr == 0xf3);
5408	/* SS */
5409	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
5410	Assert(pCtx->ss.u32Limit == 0xffff);
5411	Assert(u32SSAttr == 0xf3);
5412	/* DS */
5413	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
5414	Assert(pCtx->ds.u32Limit == 0xffff);
5415	Assert(u32DSAttr == 0xf3);
5416	/* ES */
5417	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
5418	Assert(pCtx->es.u32Limit == 0xffff);
5419	Assert(u32ESAttr == 0xf3);
5420	/* FS */
5421	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
5422	Assert(pCtx->fs.u32Limit == 0xffff);
5423	Assert(u32FSAttr == 0xf3);
5424	/* GS */
5425	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
5426	Assert(pCtx->gs.u32Limit == 0xffff);
5427	Assert(u32GSAttr == 0xf3);
5428	/* 64-bit capable CPUs. */
5429	# if HC_ARCH_BITS == 64
5430	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5431	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
5432	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
5433	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
5434	# endif
5435	}
5436	}
5437	#endif /* VBOX_STRICT */
5438
5439
5440	/**
5441	* Exports a guest segment register into the guest-state area in the VMCS.
5442	*
5443	* @returns VBox status code.
5444	* @param pVCpu The cross context virtual CPU structure.
5445	* @param pVmcsInfo The VMCS info. object.
5446	* @param iSegReg The segment register number (X86_SREG_XXX).
5447	* @param pSelReg Pointer to the segment selector.
5448	*
5449	* @remarks No-long-jump zone!!!
5450	*/
5451	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
5452	{
5453	Assert(iSegReg < X86_SREG_COUNT);
5454	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
5455	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
5456	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
5457	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
5458
5459	uint32_t u32Access = pSelReg->Attr.u;
5460	if (pVmcsInfo->RealMode.fRealOnV86Active)
5461	{
5462	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
5463	u32Access = 0xf3;
5464	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5465	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5466	RT_NOREF_PV(pVCpu);
5467	}
5468	else
5469	{
5470	/*
5471	* The way to differentiate between whether this is really a null selector or was just
5472	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
5473	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
5474	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
5475	* NULL selectors loaded in protected-mode have their attribute as 0.
5476	*/
5477	if (!u32Access)
5478	u32Access = X86DESCATTR_UNUSABLE;
5479	}
5480
5481	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
5482	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
5483	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
5484
5485	/*
5486	* Commit it to the VMCS.
5487	*/
5488	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
5489	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
5490	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
5491	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
5492	AssertRCReturn(rc, rc);
5493	return rc;
5494	}
5495
5496
5497	/**
5498	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
5499	* area in the VMCS.
5500	*
5501	* @returns VBox status code.
5502	* @param pVCpu The cross context virtual CPU structure.
5503	* @param pVmxTransient The VMX-transient structure.
5504	*
5505	* @remarks Will import guest CR0 on strict builds during validation of
5506	* segments.
5507	* @remarks No-long-jump zone!!!
5508	*/
5509	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5510	{
5511	int rc = VERR_INTERNAL_ERROR_5;
5512	PVM pVM = pVCpu->CTX_SUFF(pVM);
5513	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5514	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5515
5516	/*
5517	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
5518	*/
5519	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
5520	{
5521	#ifdef VBOX_WITH_REM
5522	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
5523	{
5524	Assert(!pVmxTransient->fIsNestedGuest);
5525	Assert(pVM->hm.s.vmx.pRealModeTSS);
5526	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
5527	if ( pVmcsInfo->fWasInRealMode
5528	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
5529	{
5530	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
5531	in real-mode (e.g. OpenBSD 4.0) */
5532	REMFlushTBs(pVM);
5533	Log4Func(("Switch to protected mode detected!\n"));
5534	pVmcsInfo->fWasInRealMode = false;
5535	}
5536	}
5537	#endif
5538	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
5539	{
5540	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
5541	if (pVmcsInfo->RealMode.fRealOnV86Active)
5542	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
5543	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
5544	AssertRCReturn(rc, rc);
5545	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
5546	}
5547
5548	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
5549	{
5550	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
5551	if (pVmcsInfo->RealMode.fRealOnV86Active)
5552	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
5553	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
5554	AssertRCReturn(rc, rc);
5555	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
5556	}
5557
5558	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
5559	{
5560	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
5561	if (pVmcsInfo->RealMode.fRealOnV86Active)
5562	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
5563	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
5564	AssertRCReturn(rc, rc);
5565	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
5566	}
5567
5568	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
5569	{
5570	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
5571	if (pVmcsInfo->RealMode.fRealOnV86Active)
5572	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
5573	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
5574	AssertRCReturn(rc, rc);
5575	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
5576	}
5577
5578	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
5579	{
5580	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
5581	if (pVmcsInfo->RealMode.fRealOnV86Active)
5582	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
5583	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
5584	AssertRCReturn(rc, rc);
5585	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
5586	}
5587
5588	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
5589	{
5590	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
5591	if (pVmcsInfo->RealMode.fRealOnV86Active)
5592	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
5593	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
5594	AssertRCReturn(rc, rc);
5595	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
5596	}
5597
5598	#ifdef VBOX_STRICT
5599	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
5600	#endif
5601	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
5602	pCtx->cs.Attr.u));
5603	}
5604
5605	/*
5606	* Guest TR.
5607	*/
5608	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
5609	{
5610	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
5611
5612	/*
5613	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
5614	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
5615	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
5616	*/
5617	uint16_t u16Sel;
5618	uint32_t u32Limit;
5619	uint64_t u64Base;
5620	uint32_t u32AccessRights;
5621	if (!pVmcsInfo->RealMode.fRealOnV86Active)
5622	{
5623	u16Sel = pCtx->tr.Sel;
5624	u32Limit = pCtx->tr.u32Limit;
5625	u64Base = pCtx->tr.u64Base;
5626	u32AccessRights = pCtx->tr.Attr.u;
5627	}
5628	else
5629	{
5630	Assert(!pVmxTransient->fIsNestedGuest);
5631	Assert(pVM->hm.s.vmx.pRealModeTSS);
5632	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
5633
5634	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
5635	RTGCPHYS GCPhys;
5636	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
5637	AssertRCReturn(rc, rc);
5638
5639	X86DESCATTR DescAttr;
5640	DescAttr.u = 0;
5641	DescAttr.n.u1Present = 1;
5642	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
5643
5644	u16Sel = 0;
5645	u32Limit = HM_VTX_TSS_SIZE;
5646	u64Base = GCPhys;
5647	u32AccessRights = DescAttr.u;
5648	}
5649
5650	/* Validate. */
5651	Assert(!(u16Sel & RT_BIT(2)));
5652	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
5653	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
5654	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
5655	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
5656	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
5657	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
5658	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
5659	Assert( (u32Limit & 0xfff) == 0xfff
5660	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
5661	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
5662	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
5663
5664	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
5665	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
5666	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
5667	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
5668	AssertRCReturn(rc, rc);
5669
5670	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
5671	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
5672	}
5673
5674	/*
5675	* Guest GDTR.
5676	*/
5677	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
5678	{
5679	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
5680
5681	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
5682	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
5683	AssertRCReturn(rc, rc);
5684
5685	/* Validate. */
5686	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5687
5688	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
5689	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
5690	}
5691
5692	/*
5693	* Guest LDTR.
5694	*/
5695	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
5696	{
5697	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
5698
5699	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
5700	uint32_t u32Access;
5701	if ( !pVmxTransient->fIsNestedGuest
5702	&& !pCtx->ldtr.Attr.u)
5703	u32Access = X86DESCATTR_UNUSABLE;
5704	else
5705	u32Access = pCtx->ldtr.Attr.u;
5706
5707	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
5708	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
5709	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
5710	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
5711	AssertRCReturn(rc, rc);
5712
5713	/* Validate. */
5714	if (!(u32Access & X86DESCATTR_UNUSABLE))
5715	{
5716	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
5717	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
5718	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
5719	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
5720	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
5721	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
5722	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
5723	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
5724	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
5725	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
5726	}
5727
5728	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
5729	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
5730	}
5731
5732	/*
5733	* Guest IDTR.
5734	*/
5735	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
5736	{
5737	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
5738
5739	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
5740	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
5741	AssertRCReturn(rc, rc);
5742
5743	/* Validate. */
5744	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5745
5746	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
5747	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
5748	}
5749
5750	return VINF_SUCCESS;
5751	}
5752
5753
5754	/**
5755	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
5756	* areas.
5757	*
5758	* These MSRs will automatically be loaded to the host CPU on every successful
5759	* VM-entry and stored from the host CPU on every successful VM-exit.
5760	*
5761	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
5762	* actual host MSR values are not- updated here for performance reasons. See
5763	* hmR0VmxExportHostMsrs().
5764	*
5765	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
5766	*
5767	* @returns VBox status code.
5768	* @param pVCpu The cross context virtual CPU structure.
5769	* @param pVmxTransient The VMX-transient structure.
5770	*
5771	* @remarks No-long-jump zone!!!
5772	*/
5773	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5774	{
5775	AssertPtr(pVCpu);
5776	AssertPtr(pVmxTransient);
5777
5778	PVM pVM = pVCpu->CTX_SUFF(pVM);
5779	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5780
5781	/*
5782	* MSRs that we use the auto-load/store MSR area in the VMCS.
5783	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
5784	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
5785	*
5786	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
5787	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
5788	* emulation, nothing to do here.
5789	*/
5790	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
5791	{
5792	if ( !pVmxTransient->fIsNestedGuest
5793	&& pVM->hm.s.fAllow64BitGuests)
5794	{
5795	#if HC_ARCH_BITS == 32
5796	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
5797	Assert(!pVmxTransient->fIsNestedGuest);
5798
5799	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
5800	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
5801	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
5802	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
5803	AssertRCReturn(rc, rc);
5804	#endif
5805	}
5806	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
5807	}
5808
5809	/*
5810	* Guest Sysenter MSRs.
5811	*/
5812	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
5813	{
5814	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
5815
5816	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
5817	{
5818	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
5819	AssertRCReturn(rc, rc);
5820	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
5821	}
5822
5823	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
5824	{
5825	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
5826	AssertRCReturn(rc, rc);
5827	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
5828	}
5829
5830	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
5831	{
5832	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
5833	AssertRCReturn(rc, rc);
5834	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
5835	}
5836	}
5837
5838	/*
5839	* Guest/host EFER MSR.
5840	*/
5841	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
5842	{
5843	/* Whether we are using the VMCS to swap the EFER MSR must have been
5844	determined earlier while exporting VM-entry/VM-exit controls. */
5845	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
5846	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
5847
5848	if (hmR0VmxShouldSwapEferMsr(pVCpu))
5849	{
5850	/*
5851	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
5852	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
5853	*/
5854	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
5855	{
5856	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
5857	AssertRCReturn(rc, rc);
5858	}
5859	else
5860	{
5861	/*
5862	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
5863	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
5864	*/
5865	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
5866	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5867	AssertRCReturn(rc, rc);
5868	}
5869	}
5870	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
5871	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
5872
5873	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
5874	}
5875
5876	/*
5877	* Other MSRs.
5878	* Speculation Control (R/W).
5879	*/
5880	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
5881	{
5882	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
5883	if (pVM->cpum.ro.GuestFeatures.fIbrs)
5884	{
5885	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
5886	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5887	AssertRCReturn(rc, rc);
5888	}
5889	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
5890	}
5891
5892	return VINF_SUCCESS;
5893	}
5894
5895
5896	/**
5897	* Selects up the appropriate function to run guest code.
5898	*
5899	* @returns VBox status code.
5900	* @param pVCpu The cross context virtual CPU structure.
5901	* @param pVmxTransient The VMX-transient structure.
5902	*
5903	* @remarks No-long-jump zone!!!
5904	*/
5905	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5906	{
5907	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5908	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5909
5910	if (CPUMIsGuestInLongModeEx(pCtx))
5911	{
5912	#ifndef VBOX_ENABLE_64_BITS_GUESTS
5913	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5914	#endif
5915	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
5916	#if HC_ARCH_BITS == 32
5917	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
5918	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
5919	{
5920	#ifdef VBOX_STRICT
5921	if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5922	{
5923	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5924	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5925	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5926	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5927	("fCtxChanged=%#RX64\n", fCtxChanged));
5928	}
5929	#endif
5930	pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
5931
5932	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
5933	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
5934	pVmcsInfo->fSwitchedTo64on32 = true;
5935	Log4Func(("Selected 64-bit switcher\n"));
5936	}
5937	#else
5938	/* 64-bit host. */
5939	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
5940	#endif
5941	}
5942	else
5943	{
5944	/* Guest is not in long mode, use the 32-bit handler. */
5945	#if HC_ARCH_BITS == 32
5946	if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
5947	&& !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
5948	&& pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5949	{
5950	# ifdef VBOX_STRICT
5951	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5952	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5953	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5954	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5955	("fCtxChanged=%#RX64\n", fCtxChanged));
5956	# endif
5957	}
5958	# ifdef VBOX_ENABLE_64_BITS_GUESTS
5959	/*
5960	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
5961	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
5962	* switcher flag now because we know the guest is in a sane state where it's safe
5963	* to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
5964	* the much faster 32-bit switcher again.
5965	*/
5966	if (!pVmcsInfo->fSwitchedTo64on32)
5967	{
5968	if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
5969	Log4Func(("Selected 32-bit switcher\n"));
5970	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5971	}
5972	else
5973	{
5974	Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
5975	if ( pVmcsInfo->RealMode.fRealOnV86Active
5976	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
5977	{
5978	pVmcsInfo->fSwitchedTo64on32 = false;
5979	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5980	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
5981	\| HM_CHANGED_VMX_ENTRY_EXIT_CTLS
5982	\| HM_CHANGED_HOST_CONTEXT);
5983	Log4Func(("Selected 32-bit switcher (safe)\n"));
5984	}
5985	}
5986	# else
5987	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5988	# endif
5989	#else
5990	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5991	#endif
5992	}
5993	Assert(pVmcsInfo->pfnStartVM);
5994	return VINF_SUCCESS;
5995	}
5996
5997
5998	/**
5999	* Wrapper for running the guest code in VT-x.
6000	*
6001	* @returns VBox status code, no informational status codes.
6002	* @param pVCpu The cross context virtual CPU structure.
6003	* @param pVmxTransient The VMX-transient structure.
6004	*
6005	* @remarks No-long-jump zone!!!
6006	*/
6007	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6008	{
6009	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6010	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6011	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6012
6013	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6014
6015	/*
6016	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6017	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6018	* callee-saved and thus the need for this XMM wrapper.
6019	*
6020	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6021	*/
6022	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6023	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6024	PVM pVM = pVCpu->CTX_SUFF(pVM);
6025	#ifdef VBOX_WITH_KERNEL_USING_XMM
6026	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6027	#else
6028	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
6029	#endif
6030	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6031	return rc;
6032	}
6033
6034
6035	/**
6036	* Reports world-switch error and dumps some useful debug info.
6037	*
6038	* @param pVCpu The cross context virtual CPU structure.
6039	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6040	* @param pVmxTransient The VMX-transient structure (only
6041	* exitReason updated).
6042	*/
6043	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6044	{
6045	Assert(pVCpu);
6046	Assert(pVmxTransient);
6047	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6048
6049	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6050	switch (rcVMRun)
6051	{
6052	case VERR_VMX_INVALID_VMXON_PTR:
6053	AssertFailed();
6054	break;
6055	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6056	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6057	{
6058	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6059	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6060	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
6061	AssertRC(rc);
6062
6063	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6064	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6065	Cannot do it here as we may have been long preempted. */
6066
6067	#ifdef VBOX_STRICT
6068	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6069	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6070	pVmxTransient->uExitReason));
6071	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6072	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6073	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6074	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6075	else
6076	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6077	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6078	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6079
6080	/* VMX control bits. */
6081	uint32_t u32Val;
6082	uint64_t u64Val;
6083	RTHCUINTREG uHCReg;
6084	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
6085	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
6086	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
6087	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
6088	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
6089	{
6090	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
6091	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
6092	}
6093	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
6094	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
6095	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
6096	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
6097	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
6098	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
6099	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
6100	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
6101	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
6102	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
6103	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
6104	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
6105	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
6106	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
6107	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
6108	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
6109	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6110	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
6111	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6112	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
6113	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
6114	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
6115	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
6116	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
6117	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
6118	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
6119	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
6120	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
6121	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
6122	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6123	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
6124	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
6125	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
6126	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6127	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6128	{
6129	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
6130	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
6131	}
6132
6133	/* Guest bits. */
6134	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
6135	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
6136	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
6137	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
6138	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
6139	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
6140	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
6141	{
6142	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
6143	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
6144	}
6145
6146	/* Host bits. */
6147	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
6148	Log4(("Host CR0 %#RHr\n", uHCReg));
6149	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
6150	Log4(("Host CR3 %#RHr\n", uHCReg));
6151	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
6152	Log4(("Host CR4 %#RHr\n", uHCReg));
6153
6154	RTGDTR HostGdtr;
6155	PCX86DESCHC pDesc;
6156	ASMGetGDTR(&HostGdtr);
6157	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
6158	Log4(("Host CS %#08x\n", u32Val));
6159	if (u32Val < HostGdtr.cbGdt)
6160	{
6161	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6162	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
6163	}
6164
6165	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
6166	Log4(("Host DS %#08x\n", u32Val));
6167	if (u32Val < HostGdtr.cbGdt)
6168	{
6169	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6170	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
6171	}
6172
6173	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
6174	Log4(("Host ES %#08x\n", u32Val));
6175	if (u32Val < HostGdtr.cbGdt)
6176	{
6177	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6178	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
6179	}
6180
6181	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
6182	Log4(("Host FS %#08x\n", u32Val));
6183	if (u32Val < HostGdtr.cbGdt)
6184	{
6185	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6186	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
6187	}
6188
6189	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
6190	Log4(("Host GS %#08x\n", u32Val));
6191	if (u32Val < HostGdtr.cbGdt)
6192	{
6193	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6194	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
6195	}
6196
6197	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
6198	Log4(("Host SS %#08x\n", u32Val));
6199	if (u32Val < HostGdtr.cbGdt)
6200	{
6201	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6202	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
6203	}
6204
6205	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
6206	Log4(("Host TR %#08x\n", u32Val));
6207	if (u32Val < HostGdtr.cbGdt)
6208	{
6209	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6210	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
6211	}
6212
6213	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
6214	Log4(("Host TR Base %#RHv\n", uHCReg));
6215	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
6216	Log4(("Host GDTR Base %#RHv\n", uHCReg));
6217	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
6218	Log4(("Host IDTR Base %#RHv\n", uHCReg));
6219	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
6220	Log4(("Host SYSENTER CS %#08x\n", u32Val));
6221	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
6222	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
6223	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
6224	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
6225	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
6226	Log4(("Host RSP %#RHv\n", uHCReg));
6227	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
6228	Log4(("Host RIP %#RHv\n", uHCReg));
6229	# if HC_ARCH_BITS == 64
6230	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6231	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6232	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6233	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6234	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6235	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6236	# endif
6237	#endif /* VBOX_STRICT */
6238	break;
6239	}
6240
6241	default:
6242	/* Impossible */
6243	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6244	break;
6245	}
6246	}
6247
6248
6249	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
6250	# ifndef VMX_USE_CACHED_VMCS_ACCESSES
6251	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
6252	# endif
6253
6254	/**
6255	* Initialize the VMCS-Read cache.
6256	*
6257	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
6258	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
6259	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
6260	* (those that have a 32-bit FULL & HIGH part).
6261	*
6262	* @param pVCpu The cross context virtual CPU structure.
6263	*/
6264	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
6265	{
6266	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
6267	do { \
6268	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
6269	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
6270	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
6271	++cReadFields; \
6272	} while (0)
6273
6274	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6275	uint32_t cReadFields = 0;
6276
6277	/*
6278	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
6279	* and serve to indicate exceptions to the rules.
6280	*/
6281
6282	/* Guest-natural selector base fields. */
6283	#if 0
6284	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
6285	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
6286	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
6287	#endif
6288	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
6289	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
6290	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
6291	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
6292	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
6293	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
6294	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
6295	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
6296	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
6297	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
6298	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
6299	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
6300	#if 0
6301	/* Unused natural width guest-state fields. */
6302	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
6303	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
6304	#endif
6305	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
6306	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
6307
6308	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
6309	these 64-bit fields (using "FULL" and "HIGH" fields). */
6310	#if 0
6311	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
6312	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
6313	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
6314	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
6315	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
6316	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
6317	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
6318	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
6319	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
6320	#endif
6321
6322	/* Natural width guest-state fields. */
6323	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
6324	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
6325
6326	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6327	{
6328	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
6329	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
6330	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
6331	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
6332	}
6333	else
6334	{
6335	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
6336	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
6337	}
6338
6339	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
6340	}
6341
6342
6343	/**
6344	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
6345	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
6346	* darwin, running 64-bit guests).
6347	*
6348	* @returns VBox status code.
6349	* @param pVCpu The cross context virtual CPU structure.
6350	* @param idxField The VMCS field encoding.
6351	* @param u64Val 16, 32 or 64-bit value.
6352	*/
6353	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6354	{
6355	int rc;
6356	switch (idxField)
6357	{
6358	/*
6359	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
6360	*/
6361	/* 64-bit Control fields. */
6362	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
6363	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
6364	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
6365	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
6366	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
6367	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
6368	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
6369	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
6370	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
6371	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
6372	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
6373	case VMX_VMCS64_CTRL_EPTP_FULL:
6374	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
6375	/* 64-bit Guest-state fields. */
6376	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
6377	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
6378	case VMX_VMCS64_GUEST_PAT_FULL:
6379	case VMX_VMCS64_GUEST_EFER_FULL:
6380	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
6381	case VMX_VMCS64_GUEST_PDPTE0_FULL:
6382	case VMX_VMCS64_GUEST_PDPTE1_FULL:
6383	case VMX_VMCS64_GUEST_PDPTE2_FULL:
6384	case VMX_VMCS64_GUEST_PDPTE3_FULL:
6385	/* 64-bit Host-state fields. */
6386	case VMX_VMCS64_HOST_PAT_FULL:
6387	case VMX_VMCS64_HOST_EFER_FULL:
6388	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
6389	{
6390	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6391	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
6392	break;
6393	}
6394
6395	/*
6396	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
6397	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
6398	*/
6399	/* Natural-width Guest-state fields. */
6400	case VMX_VMCS_GUEST_CR3:
6401	case VMX_VMCS_GUEST_ES_BASE:
6402	case VMX_VMCS_GUEST_CS_BASE:
6403	case VMX_VMCS_GUEST_SS_BASE:
6404	case VMX_VMCS_GUEST_DS_BASE:
6405	case VMX_VMCS_GUEST_FS_BASE:
6406	case VMX_VMCS_GUEST_GS_BASE:
6407	case VMX_VMCS_GUEST_LDTR_BASE:
6408	case VMX_VMCS_GUEST_TR_BASE:
6409	case VMX_VMCS_GUEST_GDTR_BASE:
6410	case VMX_VMCS_GUEST_IDTR_BASE:
6411	case VMX_VMCS_GUEST_RSP:
6412	case VMX_VMCS_GUEST_RIP:
6413	case VMX_VMCS_GUEST_SYSENTER_ESP:
6414	case VMX_VMCS_GUEST_SYSENTER_EIP:
6415	{
6416	if (!(RT_HI_U32(u64Val)))
6417	{
6418	/* If this field is 64-bit, VT-x will zero out the top bits. */
6419	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6420	}
6421	else
6422	{
6423	/* Assert that only the 32->64 switcher case should ever come here. */
6424	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
6425	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
6426	}
6427	break;
6428	}
6429
6430	default:
6431	{
6432	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
6433	pVCpu->hm.s.u32HMError = idxField;
6434	rc = VERR_INVALID_PARAMETER;
6435	break;
6436	}
6437	}
6438	AssertRCReturn(rc, rc);
6439	return rc;
6440	}
6441
6442
6443	/**
6444	* Queue up a VMWRITE by using the VMCS write cache.
6445	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
6446	*
6447	* @param pVCpu The cross context virtual CPU structure.
6448	* @param idxField The VMCS field encoding.
6449	* @param u64Val 16, 32 or 64-bit value.
6450	*/
6451	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6452	{
6453	AssertPtr(pVCpu);
6454	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6455
6456	AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
6457	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
6458
6459	/* Make sure there are no duplicates. */
6460	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
6461	{
6462	if (pCache->Write.aField[i] == idxField)
6463	{
6464	pCache->Write.aFieldVal[i] = u64Val;
6465	return VINF_SUCCESS;
6466	}
6467	}
6468
6469	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
6470	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
6471	pCache->Write.cValidEntries++;
6472	return VINF_SUCCESS;
6473	}
6474	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
6475
6476
6477	/**
6478	* Sets up the usage of TSC-offsetting and updates the VMCS.
6479	*
6480	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6481	* VMX-preemption timer.
6482	*
6483	* @returns VBox status code.
6484	* @param pVCpu The cross context virtual CPU structure.
6485	* @param pVmxTransient The VMX-transient structure.
6486	*
6487	* @remarks No-long-jump zone!!!
6488	*/
6489	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6490	{
6491	bool fOffsettedTsc;
6492	bool fParavirtTsc;
6493	uint64_t uTscOffset;
6494	PVM pVM = pVCpu->CTX_SUFF(pVM);
6495	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6496
6497	if (pVM->hm.s.vmx.fUsePreemptTimer)
6498	{
6499	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6500
6501	/* Make sure the returned values have sane upper and lower boundaries. */
6502	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6503	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6504	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6505	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6506
6507	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6508	* preemption timers here. We probably need to clamp the preemption timer,
6509	* after converting the timer value to the host. */
6510	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6511	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6512	AssertRC(rc);
6513	}
6514	else
6515	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6516
6517	if (fParavirtTsc)
6518	{
6519	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6520	information before every VM-entry, hence disable it for performance sake. */
6521	#if 0
6522	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
6523	AssertRC(rc);
6524	#endif
6525	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
6526	}
6527
6528	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
6529	if ( fOffsettedTsc
6530	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
6531	{
6532	if (pVmxTransient->fIsNestedGuest)
6533	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
6534	if (pVmcsInfo->u64TscOffset != uTscOffset)
6535	{
6536	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
6537	AssertRC(rc);
6538	pVmcsInfo->u64TscOffset = uTscOffset;
6539	}
6540
6541	if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
6542	{
6543	uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
6544	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6545	AssertRC(rc);
6546	pVmcsInfo->u32ProcCtls = uProcCtls;
6547	}
6548	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
6549	}
6550	else
6551	{
6552	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
6553	if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
6554	{
6555	uProcCtls \|= VMX_PROC_CTLS_RDTSC_EXIT;
6556	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6557	AssertRC(rc);
6558	pVmcsInfo->u32ProcCtls = uProcCtls;
6559	}
6560	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
6561	}
6562	}
6563
6564
6565	/**
6566	* Gets the IEM exception flags for the specified vector and IDT vectoring /
6567	* VM-exit interruption info type.
6568	*
6569	* @returns The IEM exception flags.
6570	* @param uVector The event vector.
6571	* @param uVmxEventType The VMX event type.
6572	*
6573	* @remarks This function currently only constructs flags required for
6574	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
6575	* and CR2 aspects of an exception are not included).
6576	*/
6577	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
6578	{
6579	uint32_t fIemXcptFlags;
6580	switch (uVmxEventType)
6581	{
6582	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6583	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6584	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6585	break;
6586
6587	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6588	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6589	break;
6590
6591	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6592	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
6593	break;
6594
6595	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
6596	{
6597	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6598	if (uVector == X86_XCPT_BP)
6599	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
6600	else if (uVector == X86_XCPT_OF)
6601	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
6602	else
6603	{
6604	fIemXcptFlags = 0;
6605	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
6606	}
6607	break;
6608	}
6609
6610	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6611	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6612	break;
6613
6614	default:
6615	fIemXcptFlags = 0;
6616	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
6617	break;
6618	}
6619	return fIemXcptFlags;
6620	}
6621
6622
6623	/**
6624	* Sets an event as a pending event to be injected into the guest.
6625	*
6626	* @param pVCpu The cross context virtual CPU structure.
6627	* @param u32IntInfo The VM-entry interruption-information field.
6628	* @param cbInstr The VM-entry instruction length in bytes (for software
6629	* interrupts, exceptions and privileged software
6630	* exceptions).
6631	* @param u32ErrCode The VM-entry exception error code.
6632	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
6633	* page-fault.
6634	*/
6635	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
6636	RTGCUINTPTR GCPtrFaultAddress)
6637	{
6638	Assert(!pVCpu->hm.s.Event.fPending);
6639	pVCpu->hm.s.Event.fPending = true;
6640	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
6641	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
6642	pVCpu->hm.s.Event.cbInstr = cbInstr;
6643	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
6644	}
6645
6646
6647	/**
6648	* Sets an external interrupt as pending-for-injection into the VM.
6649	*
6650	* @param pVCpu The cross context virtual CPU structure.
6651	* @param u8Interrupt The external interrupt vector.
6652	*/
6653	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
6654	{
6655	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
6656	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6657	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6658	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6659	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6660	}
6661
6662
6663	/**
6664	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
6665	*
6666	* @param pVCpu The cross context virtual CPU structure.
6667	*/
6668	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
6669	{
6670	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
6671	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
6672	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6673	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6674	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6675	}
6676
6677
6678	/**
6679	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
6680	*
6681	* @param pVCpu The cross context virtual CPU structure.
6682	*/
6683	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
6684	{
6685	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
6686	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6687	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6688	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6689	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6690	}
6691
6692
6693	/**
6694	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
6695	*
6696	* @param pVCpu The cross context virtual CPU structure.
6697	*/
6698	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
6699	{
6700	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
6701	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6702	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6703	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6704	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6705	}
6706
6707
6708	/**
6709	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
6710	*
6711	* @param pVCpu The cross context virtual CPU structure.
6712	*/
6713	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
6714	{
6715	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
6716	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6717	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6718	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6719	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6720	}
6721
6722
6723	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6724	/**
6725	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
6726	*
6727	* @param pVCpu The cross context virtual CPU structure.
6728	* @param u32ErrCode The error code for the general-protection exception.
6729	*/
6730	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
6731	{
6732	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
6733	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6734	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6735	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6736	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6737	}
6738
6739
6740	/**
6741	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
6742	*
6743	* @param pVCpu The cross context virtual CPU structure.
6744	* @param u32ErrCode The error code for the stack exception.
6745	*/
6746	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
6747	{
6748	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
6749	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6750	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6751	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6752	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6753	}
6754
6755
6756	/**
6757	* Decodes the memory operand of an instruction that caused a VM-exit.
6758	*
6759	* The VM-exit qualification field provides the displacement field for memory
6760	* operand instructions, if any.
6761	*
6762	* @returns Strict VBox status code (i.e. informational status codes too).
6763	* @retval VINF_SUCCESS if the operand was successfully decoded.
6764	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
6765	* operand.
6766	* @param pVCpu The cross context virtual CPU structure.
6767	* @param uExitInstrInfo The VM-exit instruction information field.
6768	* @param enmMemAccess The memory operand's access type (read or write).
6769	* @param GCPtrDisp The instruction displacement field, if any. For
6770	* RIP-relative addressing pass RIP + displacement here.
6771	* @param pGCPtrMem Where to store the effective destination memory address.
6772	*/
6773	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
6774	PRTGCPTR pGCPtrMem)
6775	{
6776	Assert(pGCPtrMem);
6777	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
6778	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
6779	\| CPUMCTX_EXTRN_CR0);
6780
6781	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
6782	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
6783	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
6784
6785	VMXEXITINSTRINFO ExitInstrInfo;
6786	ExitInstrInfo.u = uExitInstrInfo;
6787	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
6788	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
6789	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
6790	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
6791	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
6792	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
6793	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
6794	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
6795	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
6796
6797	/*
6798	* Validate instruction information.
6799	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
6800	*/
6801	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
6802	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
6803	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
6804	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
6805	AssertLogRelMsgReturn(fIsMemOperand,
6806	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
6807
6808	/*
6809	* Compute the complete effective address.
6810	*
6811	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
6812	* See AMD spec. 4.5.2 "Segment Registers".
6813	*/
6814	RTGCPTR GCPtrMem = GCPtrDisp;
6815	if (fBaseRegValid)
6816	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
6817	if (fIdxRegValid)
6818	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
6819
6820	RTGCPTR const GCPtrOff = GCPtrMem;
6821	if ( !fIsLongMode
6822	\|\| iSegReg >= X86_SREG_FS)
6823	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
6824	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
6825
6826	/*
6827	* Validate effective address.
6828	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
6829	*/
6830	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
6831	Assert(cbAccess > 0);
6832	if (fIsLongMode)
6833	{
6834	if (X86_IS_CANONICAL(GCPtrMem))
6835	{
6836	*pGCPtrMem = GCPtrMem;
6837	return VINF_SUCCESS;
6838	}
6839
6840	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
6841	* "Data Limit Checks in 64-bit Mode". */
6842	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
6843	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6844	return VINF_HM_PENDING_XCPT;
6845	}
6846
6847	/*
6848	* This is a watered down version of iemMemApplySegment().
6849	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
6850	* and segment CPL/DPL checks are skipped.
6851	*/
6852	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
6853	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
6854	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
6855
6856	/* Check if the segment is present and usable. */
6857	if ( pSel->Attr.n.u1Present
6858	&& !pSel->Attr.n.u1Unusable)
6859	{
6860	Assert(pSel->Attr.n.u1DescType);
6861	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
6862	{
6863	/* Check permissions for the data segment. */
6864	if ( enmMemAccess == VMXMEMACCESS_WRITE
6865	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
6866	{
6867	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6868	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
6869	return VINF_HM_PENDING_XCPT;
6870	}
6871
6872	/* Check limits if it's a normal data segment. */
6873	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
6874	{
6875	if ( GCPtrFirst32 > pSel->u32Limit
6876	\|\| GCPtrLast32 > pSel->u32Limit)
6877	{
6878	Log4Func(("Data segment limit exceeded."
6879	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6880	GCPtrLast32, pSel->u32Limit));
6881	if (iSegReg == X86_SREG_SS)
6882	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6883	else
6884	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6885	return VINF_HM_PENDING_XCPT;
6886	}
6887	}
6888	else
6889	{
6890	/* Check limits if it's an expand-down data segment.
6891	Note! The upper boundary is defined by the B bit, not the G bit! */
6892	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
6893	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
6894	{
6895	Log4Func(("Expand-down data segment limit exceeded."
6896	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6897	GCPtrLast32, pSel->u32Limit));
6898	if (iSegReg == X86_SREG_SS)
6899	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6900	else
6901	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6902	return VINF_HM_PENDING_XCPT;
6903	}
6904	}
6905	}
6906	else
6907	{
6908	/* Check permissions for the code segment. */
6909	if ( enmMemAccess == VMXMEMACCESS_WRITE
6910	\|\| ( enmMemAccess == VMXMEMACCESS_READ
6911	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
6912	{
6913	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6914	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6915	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6916	return VINF_HM_PENDING_XCPT;
6917	}
6918
6919	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6920	if ( GCPtrFirst32 > pSel->u32Limit
6921	\|\| GCPtrLast32 > pSel->u32Limit)
6922	{
6923	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6924	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6925	if (iSegReg == X86_SREG_SS)
6926	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6927	else
6928	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6929	return VINF_HM_PENDING_XCPT;
6930	}
6931	}
6932	}
6933	else
6934	{
6935	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6936	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6937	return VINF_HM_PENDING_XCPT;
6938	}
6939
6940	*pGCPtrMem = GCPtrMem;
6941	return VINF_SUCCESS;
6942	}
6943
6944
6945	/**
6946	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6947	* guest attempting to execute a VMX instruction.
6948	*
6949	* @returns Strict VBox status code (i.e. informational status codes too).
6950	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
6951	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
6952	*
6953	* @param pVCpu The cross context virtual CPU structure.
6954	* @param uExitReason The VM-exit reason.
6955	*
6956	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
6957	* @remarks No-long-jump zone!!!
6958	*/
6959	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
6960	{
6961	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
6962	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
6963
6964	if ( CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx)
6965	\|\| ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6966	&& !CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)))
6967	{
6968	Log4Func(("In real/v86-mode or long-mode outside 64-bit code segment -> #UD\n"));
6969	hmR0VmxSetPendingXcptUD(pVCpu);
6970	return VINF_HM_PENDING_XCPT;
6971	}
6972
6973	if (uExitReason == VMX_EXIT_VMXON)
6974	{
6975	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
6976
6977	/*
6978	* We check CR4.VMXE because it is required to be always set while in VMX operation
6979	* by physical CPUs and our CR4 read shadow is only consulted when executing specific
6980	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
6981	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
6982	*/
6983	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
6984	{
6985	Log4Func(("CR4.VMXE is not set -> #UD\n"));
6986	hmR0VmxSetPendingXcptUD(pVCpu);
6987	return VINF_HM_PENDING_XCPT;
6988	}
6989	}
6990	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
6991	{
6992	/*
6993	* The guest has not entered VMX operation but attempted to execute a VMX instruction
6994	* (other than VMXON), we need to raise a #UD.
6995	*/
6996	Log4Func(("Not in VMX root mode -> #UD\n"));
6997	hmR0VmxSetPendingXcptUD(pVCpu);
6998	return VINF_HM_PENDING_XCPT;
6999	}
7000
7001	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
7002	{
7003	/*
7004	* The nested-guest attempted to execute a VMX instruction, cause a VM-exit and let
7005	* the guest hypervisor deal with it.
7006	*/
7007	/** @todo NSTVMX: Trigger a VM-exit */
7008	}
7009
7010	/*
7011	* VMX instructions require CPL 0 except in VMX non-root mode where the VM-exit intercept
7012	* (above) takes preceedence over the CPL check.
7013	*/
7014	if (CPUMGetGuestCPL(pVCpu) > 0)
7015	{
7016	Log4Func(("CPL > 0 -> #GP(0)\n"));
7017	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7018	return VINF_HM_PENDING_XCPT;
7019	}
7020
7021	return VINF_SUCCESS;
7022	}
7023	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7024
7025
7026	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7027	{
7028	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7029
7030	/*
7031	* If VT-x marks the segment as unusable, most other bits remain undefined:
7032	* - For CS the L, D and G bits have meaning.
7033	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7034	* - For the remaining data segments no bits are defined.
7035	*
7036	* The present bit and the unusable bit has been observed to be set at the
7037	* same time (the selector was supposed to be invalid as we started executing
7038	* a V8086 interrupt in ring-0).
7039	*
7040	* What should be important for the rest of the VBox code, is that the P bit is
7041	* cleared. Some of the other VBox code recognizes the unusable bit, but
7042	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7043	* safe side here, we'll strip off P and other bits we don't care about. If
7044	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7045	*
7046	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7047	*/
7048	#ifdef VBOX_STRICT
7049	uint32_t const uAttr = pSelReg->Attr.u;
7050	#endif
7051
7052	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7053	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7054	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7055
7056	#ifdef VBOX_STRICT
7057	VMMRZCallRing3Disable(pVCpu);
7058	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7059	# ifdef DEBUG_bird
7060	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7061	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7062	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7063	# endif
7064	VMMRZCallRing3Enable(pVCpu);
7065	NOREF(uAttr);
7066	#endif
7067	RT_NOREF2(pVCpu, idxSel);
7068	}
7069
7070
7071	/**
7072	* Imports a guest segment register from the current VMCS into the guest-CPU
7073	* context.
7074	*
7075	* @returns VBox status code.
7076	* @param pVCpu The cross context virtual CPU structure.
7077	* @param iSegReg The segment register number (X86_SREG_XXX).
7078	*
7079	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7080	* do not log!
7081	*/
7082	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7083	{
7084	Assert(iSegReg < X86_SREG_COUNT);
7085
7086	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7087	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7088	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7089	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
7090	uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
7091	#else
7092	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7093	#endif
7094	uint64_t u64Base;
7095	uint32_t u32Sel, u32Limit, u32Attr;
7096	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7097	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7098	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7099	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
7100	if (RT_SUCCESS(rc))
7101	{
7102	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7103	pSelReg->Sel = u32Sel;
7104	pSelReg->ValidSel = u32Sel;
7105	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7106	pSelReg->u32Limit = u32Limit;
7107	pSelReg->u64Base = u64Base;
7108	pSelReg->Attr.u = u32Attr;
7109	if (u32Attr & X86DESCATTR_UNUSABLE)
7110	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7111	}
7112	return rc;
7113	}
7114
7115
7116	/**
7117	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7118	*
7119	* @returns VBox status code.
7120	* @param pVCpu The cross context virtual CPU structure.
7121	*
7122	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7123	* do not log!
7124	*/
7125	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7126	{
7127	uint64_t u64Base;
7128	uint32_t u32Sel, u32Limit, u32Attr;
7129	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7130	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7131	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7132	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7133	if (RT_SUCCESS(rc))
7134	{
7135	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7136	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7137	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7138	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7139	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7140	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7141	if (u32Attr & X86DESCATTR_UNUSABLE)
7142	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7143	}
7144	return rc;
7145	}
7146
7147
7148	/**
7149	* Imports the guest TR from the current VMCS into the guest-CPU context.
7150	*
7151	* @returns VBox status code.
7152	* @param pVCpu The cross context virtual CPU structure.
7153	*
7154	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7155	* do not log!
7156	*/
7157	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7158	{
7159	uint32_t u32Sel, u32Limit, u32Attr;
7160	uint64_t u64Base;
7161	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7162	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7163	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7164	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7165	AssertRCReturn(rc, rc);
7166
7167	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7168	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7169	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7170	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7171	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7172	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7173	/* TR is the only selector that can never be unusable. */
7174	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7175	return VINF_SUCCESS;
7176	}
7177
7178
7179	/**
7180	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7181	*
7182	* @returns VBox status code.
7183	* @param pVCpu The cross context virtual CPU structure.
7184	*
7185	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7186	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7187	* instead!!!
7188	*/
7189	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7190	{
7191	uint64_t u64Val;
7192	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7193	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7194	{
7195	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7196	if (RT_SUCCESS(rc))
7197	{
7198	pCtx->rip = u64Val;
7199	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7200	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7201	}
7202	return rc;
7203	}
7204	return VINF_SUCCESS;
7205	}
7206
7207
7208	/**
7209	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7210	*
7211	* @returns VBox status code.
7212	* @param pVCpu The cross context virtual CPU structure.
7213	* @param pVmcsInfo The VMCS info. object.
7214	*
7215	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7216	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7217	* instead!!!
7218	*/
7219	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7220	{
7221	uint32_t u32Val;
7222	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7223	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7224	{
7225	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7226	if (RT_SUCCESS(rc))
7227	{
7228	pCtx->eflags.u32 = u32Val;
7229
7230	/* Restore eflags for real-on-v86-mode hack. */
7231	if (pVmcsInfo->RealMode.fRealOnV86Active)
7232	{
7233	pCtx->eflags.Bits.u1VM = 0;
7234	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7235	}
7236	}
7237	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7238	return rc;
7239	}
7240	return VINF_SUCCESS;
7241	}
7242
7243
7244	/**
7245	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7246	* context.
7247	*
7248	* @returns VBox status code.
7249	* @param pVCpu The cross context virtual CPU structure.
7250	* @param pVmcsInfo The VMCS info. object.
7251	*
7252	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7253	* do not log!
7254	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7255	* instead!!!
7256	*/
7257	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7258	{
7259	uint32_t u32Val;
7260	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7261	if (RT_SUCCESS(rc))
7262	{
7263	if (!u32Val)
7264	{
7265	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7266	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7267
7268	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7269	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7270	}
7271	else
7272	{
7273	/*
7274	* We must import RIP here to set our EM interrupt-inhibited state.
7275	* We also import RFLAGS as our code that evaluates pending interrupts
7276	* before VM-entry requires it.
7277	*/
7278	rc = hmR0VmxImportGuestRip(pVCpu);
7279	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7280	if (RT_SUCCESS(rc))
7281	{
7282	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7283	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7284	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7285	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7286
7287	if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
7288	{
7289	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7290	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
7291	}
7292	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7293	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7294	}
7295	}
7296	}
7297	return rc;
7298	}
7299
7300
7301	/**
7302	* Worker for VMXR0ImportStateOnDemand.
7303	*
7304	* @returns VBox status code.
7305	* @param pVCpu The cross context virtual CPU structure.
7306	* @param pVmcsInfo The VMCS info. object.
7307	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7308	*/
7309	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7310	{
7311	#define VMXLOCAL_BREAK_RC(a_rc) \
7312	if (RT_SUCCESS(a_rc)) \
7313	{ } \
7314	else \
7315	break
7316
7317	int rc = VINF_SUCCESS;
7318	PVM pVM = pVCpu->CTX_SUFF(pVM);
7319	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7320	uint64_t u64Val;
7321	uint32_t u32Val;
7322
7323	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
7324
7325	/*
7326	* We disable interrupts to make the updating of the state and in particular
7327	* the fExtrn modification atomic wrt to preemption hooks.
7328	*/
7329	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7330
7331	fWhat &= pCtx->fExtrn;
7332	if (fWhat)
7333	{
7334	do
7335	{
7336	if (fWhat & CPUMCTX_EXTRN_RIP)
7337	{
7338	rc = hmR0VmxImportGuestRip(pVCpu);
7339	VMXLOCAL_BREAK_RC(rc);
7340	}
7341
7342	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7343	{
7344	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7345	VMXLOCAL_BREAK_RC(rc);
7346	}
7347
7348	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7349	{
7350	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7351	VMXLOCAL_BREAK_RC(rc);
7352	}
7353
7354	if (fWhat & CPUMCTX_EXTRN_RSP)
7355	{
7356	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
7357	VMXLOCAL_BREAK_RC(rc);
7358	pCtx->rsp = u64Val;
7359	}
7360
7361	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7362	{
7363	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7364	if (fWhat & CPUMCTX_EXTRN_CS)
7365	{
7366	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7367	rc \|= hmR0VmxImportGuestRip(pVCpu);
7368	if (fRealOnV86Active)
7369	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7370	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7371	}
7372	if (fWhat & CPUMCTX_EXTRN_SS)
7373	{
7374	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7375	if (fRealOnV86Active)
7376	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7377	}
7378	if (fWhat & CPUMCTX_EXTRN_DS)
7379	{
7380	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7381	if (fRealOnV86Active)
7382	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7383	}
7384	if (fWhat & CPUMCTX_EXTRN_ES)
7385	{
7386	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7387	if (fRealOnV86Active)
7388	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7389	}
7390	if (fWhat & CPUMCTX_EXTRN_FS)
7391	{
7392	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7393	if (fRealOnV86Active)
7394	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7395	}
7396	if (fWhat & CPUMCTX_EXTRN_GS)
7397	{
7398	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7399	if (fRealOnV86Active)
7400	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7401	}
7402	VMXLOCAL_BREAK_RC(rc);
7403	}
7404
7405	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7406	{
7407	if (fWhat & CPUMCTX_EXTRN_LDTR)
7408	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
7409
7410	if (fWhat & CPUMCTX_EXTRN_GDTR)
7411	{
7412	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
7413	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
7414	pCtx->gdtr.pGdt = u64Val;
7415	pCtx->gdtr.cbGdt = u32Val;
7416	}
7417
7418	/* Guest IDTR. */
7419	if (fWhat & CPUMCTX_EXTRN_IDTR)
7420	{
7421	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
7422	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
7423	pCtx->idtr.pIdt = u64Val;
7424	pCtx->idtr.cbIdt = u32Val;
7425	}
7426
7427	/* Guest TR. */
7428	if (fWhat & CPUMCTX_EXTRN_TR)
7429	{
7430	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7431	don't need to import that one. */
7432	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7433	rc \|= hmR0VmxImportGuestTr(pVCpu);
7434	}
7435	VMXLOCAL_BREAK_RC(rc);
7436	}
7437
7438	if (fWhat & CPUMCTX_EXTRN_DR7)
7439	{
7440	if (!pVCpu->hm.s.fUsingHyperDR7)
7441	{
7442	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
7443	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
7444	VMXLOCAL_BREAK_RC(rc);
7445	pCtx->dr[7] = u32Val;
7446	}
7447	}
7448
7449	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7450	{
7451	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
7452	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
7453	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
7454	pCtx->SysEnter.cs = u32Val;
7455	VMXLOCAL_BREAK_RC(rc);
7456	}
7457
7458	#if HC_ARCH_BITS == 64
7459	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7460	{
7461	if ( pVM->hm.s.fAllow64BitGuests
7462	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7463	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7464	}
7465
7466	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7467	{
7468	if ( pVM->hm.s.fAllow64BitGuests
7469	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7470	{
7471	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7472	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7473	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7474	}
7475	}
7476	#endif
7477
7478	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7479	#if HC_ARCH_BITS == 32
7480	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
7481	#endif
7482	)
7483	{
7484	PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7485	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7486	Assert(cMsrs == 0 \|\| pMsr != NULL);
7487	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
7488	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
7489	{
7490	switch (pMsr->u32Msr)
7491	{
7492	#if HC_ARCH_BITS == 32
7493	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
7494	case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
7495	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
7496	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
7497	#endif
7498	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
7499	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
7500	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7501
7502	default:
7503	{
7504	pVCpu->hm.s.u32HMError = pMsr->u32Msr;
7505	ASMSetFlags(fEFlags);
7506	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
7507	cMsrs));
7508	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7509	}
7510	}
7511	}
7512	}
7513
7514	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7515	{
7516	uint64_t u64Shadow;
7517	if (fWhat & CPUMCTX_EXTRN_CR0)
7518	{
7519	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7520	* remove when we drop 32-bit host w/ 64-bit host support, see
7521	* @bugref{9180#c39}. */
7522	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7523	#if HC_ARCH_BITS == 32
7524	uint32_t u32Shadow;
7525	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
7526	u64Shadow = u32Shadow;
7527	#else
7528	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
7529	#endif
7530	VMXLOCAL_BREAK_RC(rc);
7531	u64Val = u32Val;
7532	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7533	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7534	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7535	CPUMSetGuestCR0(pVCpu, u64Val);
7536	VMMRZCallRing3Enable(pVCpu);
7537	}
7538
7539	if (fWhat & CPUMCTX_EXTRN_CR4)
7540	{
7541	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7542	* remove when we drop 32-bit host w/ 64-bit host support, see
7543	* @bugref{9180#c39}. */
7544	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7545	#if HC_ARCH_BITS == 32
7546	uint32_t u32Shadow;
7547	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
7548	u64Shadow = u32Shadow;
7549	#else
7550	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
7551	#endif
7552	VMXLOCAL_BREAK_RC(rc);
7553	u64Val = u32Val;
7554	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
7555	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7556	pCtx->cr4 = u64Val;
7557	}
7558
7559	if (fWhat & CPUMCTX_EXTRN_CR3)
7560	{
7561	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7562	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7563	\|\| ( pVM->hm.s.fNestedPaging
7564	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7565	{
7566	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
7567	VMXLOCAL_BREAK_RC(rc);
7568	if (pCtx->cr3 != u64Val)
7569	{
7570	pCtx->cr3 = u64Val;
7571	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7572	}
7573
7574	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7575	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7576	if (CPUMIsGuestInPAEModeEx(pCtx))
7577	{
7578	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
7579	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
7580	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
7581	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
7582	VMXLOCAL_BREAK_RC(rc);
7583	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7584	}
7585	}
7586	}
7587	}
7588	} while (0);
7589
7590	if (RT_SUCCESS(rc))
7591	{
7592	/* Update fExtrn. */
7593	pCtx->fExtrn &= ~fWhat;
7594
7595	/* If everything has been imported, clear the HM keeper bit. */
7596	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7597	{
7598	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7599	Assert(!pCtx->fExtrn);
7600	}
7601	}
7602	}
7603	else
7604	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7605
7606	ASMSetFlags(fEFlags);
7607
7608	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7609
7610	if (RT_SUCCESS(rc))
7611	{ /* likely */ }
7612	else
7613	return rc;
7614
7615	/*
7616	* Honor any pending CR3 updates.
7617	*
7618	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
7619	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7620	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7621	*
7622	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7623	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7624	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7625	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7626	*
7627	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7628	*/
7629	if (VMMRZCallRing3IsEnabled(pVCpu))
7630	{
7631	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7632	{
7633	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7634	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7635	}
7636
7637	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7638	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7639
7640	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7641	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7642	}
7643
7644	return VINF_SUCCESS;
7645	#undef VMXLOCAL_BREAK_RC
7646	}
7647
7648
7649	/**
7650	* Saves the guest state from the VMCS into the guest-CPU context.
7651	*
7652	* @returns VBox status code.
7653	* @param pVCpu The cross context virtual CPU structure.
7654	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7655	*/
7656	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
7657	{
7658	PCVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7659	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7660	}
7661
7662
7663	/**
7664	* Check per-VM and per-VCPU force flag actions that require us to go back to
7665	* ring-3 for one reason or another.
7666	*
7667	* @returns Strict VBox status code (i.e. informational status codes too)
7668	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7669	* ring-3.
7670	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7671	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7672	* interrupts)
7673	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7674	* all EMTs to be in ring-3.
7675	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7676	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7677	* to the EM loop.
7678	*
7679	* @param pVCpu The cross context virtual CPU structure.
7680	* @param fStepping Whether we are single-stepping the guest using the
7681	* hypervisor debugger.
7682	*/
7683	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
7684	{
7685	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7686
7687	/*
7688	* Update pending interrupts into the APIC's IRR.
7689	*/
7690	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7691	APICUpdatePendingInterrupts(pVCpu);
7692
7693	/*
7694	* Anything pending? Should be more likely than not if we're doing a good job.
7695	*/
7696	PVM pVM = pVCpu->CTX_SUFF(pVM);
7697	if ( !fStepping
7698	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7699	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7700	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7701	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7702	return VINF_SUCCESS;
7703
7704	/* Pending PGM C3 sync. */
7705	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7706	{
7707	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7708	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7709	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7710	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7711	if (rcStrict2 != VINF_SUCCESS)
7712	{
7713	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7714	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7715	return rcStrict2;
7716	}
7717	}
7718
7719	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7720	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7721	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7722	{
7723	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7724	int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7725	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7726	return rc2;
7727	}
7728
7729	/* Pending VM request packets, such as hardware interrupts. */
7730	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7731	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7732	{
7733	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7734	return VINF_EM_PENDING_REQUEST;
7735	}
7736
7737	/* Pending PGM pool flushes. */
7738	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7739	{
7740	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7741	return VINF_PGM_POOL_FLUSH_PENDING;
7742	}
7743
7744	/* Pending DMA requests. */
7745	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7746	{
7747	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7748	return VINF_EM_RAW_TO_R3;
7749	}
7750
7751	return VINF_SUCCESS;
7752	}
7753
7754
7755	/**
7756	* Converts any TRPM trap into a pending HM event. This is typically used when
7757	* entering from ring-3 (not longjmp returns).
7758	*
7759	* @param pVCpu The cross context virtual CPU structure.
7760	*/
7761	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7762	{
7763	Assert(TRPMHasTrap(pVCpu));
7764	Assert(!pVCpu->hm.s.Event.fPending);
7765
7766	uint8_t uVector;
7767	TRPMEVENT enmTrpmEvent;
7768	RTGCUINT uErrCode;
7769	RTGCUINTPTR GCPtrFaultAddress;
7770	uint8_t cbInstr;
7771
7772	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7773	AssertRC(rc);
7774
7775	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7776	uint32_t u32IntInfo = uVector \| VMX_EXIT_INT_INFO_VALID;
7777	if (enmTrpmEvent == TRPM_TRAP)
7778	{
7779	/** @todo r=ramshankar: TRPM currently offers no way to determine a \#DB that was
7780	* generated using INT1 (ICEBP). */
7781	switch (uVector)
7782	{
7783	case X86_XCPT_NMI:
7784	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7785	break;
7786
7787	case X86_XCPT_BP:
7788	case X86_XCPT_OF:
7789	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7790	break;
7791
7792	case X86_XCPT_PF:
7793	case X86_XCPT_DF:
7794	case X86_XCPT_TS:
7795	case X86_XCPT_NP:
7796	case X86_XCPT_SS:
7797	case X86_XCPT_GP:
7798	case X86_XCPT_AC:
7799	u32IntInfo \|= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7800	RT_FALL_THRU();
7801	default:
7802	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7803	break;
7804	}
7805	}
7806	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7807	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7808	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7809	{
7810	switch (uVector)
7811	{
7812	case X86_XCPT_BP:
7813	case X86_XCPT_OF:
7814	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7815	break;
7816
7817	default:
7818	Assert(uVector == X86_XCPT_DB);
7819	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7820	break;
7821	}
7822	}
7823	else
7824	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7825
7826	rc = TRPMResetTrap(pVCpu);
7827	AssertRC(rc);
7828	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7829	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7830
7831	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7832	}
7833
7834
7835	/**
7836	* Converts the pending HM event into a TRPM trap.
7837	*
7838	* @param pVCpu The cross context virtual CPU structure.
7839	*/
7840	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7841	{
7842	Assert(pVCpu->hm.s.Event.fPending);
7843
7844	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7845	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7846	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
7847	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7848
7849	/* If a trap was already pending, we did something wrong! */
7850	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7851
7852	/** @todo Use HMVmxEventToTrpmEventType() later. */
7853	TRPMEVENT enmTrapType;
7854	switch (uVectorType)
7855	{
7856	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7857	enmTrapType = TRPM_HARDWARE_INT;
7858	break;
7859
7860	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7861	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7862	enmTrapType = TRPM_TRAP;
7863	break;
7864
7865	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT: /* #DB (INT1/ICEBP). */
7866	Assert(uVector == X86_XCPT_DB);
7867	enmTrapType = TRPM_SOFTWARE_INT;
7868	break;
7869
7870	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP (INT3) and #OF (INTO) */
7871	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7872	enmTrapType = TRPM_SOFTWARE_INT;
7873	break;
7874
7875	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7876	enmTrapType = TRPM_SOFTWARE_INT;
7877	break;
7878
7879	default:
7880	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7881	enmTrapType = TRPM_32BIT_HACK;
7882	break;
7883	}
7884
7885	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7886
7887	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7888	AssertRC(rc);
7889
7890	if (fErrorCodeValid)
7891	TRPMSetErrorCode(pVCpu, uErrorCode);
7892
7893	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7894	&& uVector == X86_XCPT_PF)
7895	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7896	else if (enmTrapType == TRPM_SOFTWARE_INT)
7897	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7898
7899	/* We're now done converting the pending event. */
7900	pVCpu->hm.s.Event.fPending = false;
7901	}
7902
7903
7904	/**
7905	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7906	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7907	*
7908	* @param pVCpu The cross context virtual CPU structure.
7909	* @param pVmcsInfo The VMCS info. object.
7910	*/
7911	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
7912	{
7913	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7914	{
7915	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7916	{
7917	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7918	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7919	AssertRC(rc);
7920	}
7921	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
7922	}
7923
7924
7925	/**
7926	* Clears the interrupt-window exiting control in the VMCS.
7927	*
7928	* @param pVmcsInfo The VMCS info. object.
7929	*/
7930	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7931	{
7932	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7933	{
7934	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7935	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7936	}
7937	return VINF_SUCCESS;
7938	}
7939
7940
7941	/**
7942	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7943	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7944	*
7945	* @param pVCpu The cross context virtual CPU structure.
7946	* @param pVmcsInfo The VMCS info. object.
7947	*/
7948	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
7949	{
7950	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7951	{
7952	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7953	{
7954	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7955	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7956	AssertRC(rc);
7957	Log4Func(("Setup NMI-window exiting\n"));
7958	}
7959	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7960	}
7961
7962
7963	/**
7964	* Clears the NMI-window exiting control in the VMCS.
7965	*
7966	* @param pVmcsInfo The VMCS info. object.
7967	*/
7968	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7969	{
7970	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7971	{
7972	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7973	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7974	}
7975	return VINF_SUCCESS;
7976	}
7977
7978
7979	/**
7980	* Does the necessary state syncing before returning to ring-3 for any reason
7981	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7982	*
7983	* @returns VBox status code.
7984	* @param pVCpu The cross context virtual CPU structure.
7985	* @param fImportState Whether to import the guest state from the VMCS back
7986	* to the guest-CPU context.
7987	*
7988	* @remarks No-long-jmp zone!!!
7989	*/
7990	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
7991	{
7992	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7993	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7994
7995	RTCPUID idCpu = RTMpCpuId();
7996	Log4Func(("HostCpuId=%u\n", idCpu));
7997
7998	/*
7999	* !!! IMPORTANT !!!
8000	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
8001	*/
8002
8003	/* Save the guest state if necessary. */
8004	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8005	if (fImportState)
8006	{
8007	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8008	AssertRCReturn(rc, rc);
8009	}
8010
8011	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8012	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8013	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8014
8015	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8016	#ifdef VBOX_STRICT
8017	if (CPUMIsHyperDebugStateActive(pVCpu))
8018	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8019	#endif
8020	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8021	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8022	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8023
8024	#if HC_ARCH_BITS == 64
8025	/* Restore host-state bits that VT-x only restores partially. */
8026	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8027	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8028	{
8029	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8030	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8031	}
8032	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8033	#endif
8034
8035	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8036	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8037	{
8038	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8039	if (!fImportState)
8040	{
8041	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8042	AssertRCReturn(rc, rc);
8043	}
8044	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8045	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8046	}
8047	else
8048	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8049
8050	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8051	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8052
8053	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8054	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8055	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8056	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8057	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8058	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8059	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8060	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8061	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8062
8063	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8064
8065	/** @todo This partially defeats the purpose of having preemption hooks.
8066	* The problem is, deregistering the hooks should be moved to a place that
8067	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8068	* context.
8069	*/
8070	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8071	AssertRCReturn(rc, rc);
8072
8073	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8074	NOREF(idCpu);
8075	return VINF_SUCCESS;
8076	}
8077
8078
8079	/**
8080	* Leaves the VT-x session.
8081	*
8082	* @returns VBox status code.
8083	* @param pVCpu The cross context virtual CPU structure.
8084	*
8085	* @remarks No-long-jmp zone!!!
8086	*/
8087	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8088	{
8089	HM_DISABLE_PREEMPT(pVCpu);
8090	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8091	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8092	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8093
8094	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8095	and done this from the VMXR0ThreadCtxCallback(). */
8096	if (!pVCpu->hm.s.fLeaveDone)
8097	{
8098	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8099	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8100	pVCpu->hm.s.fLeaveDone = true;
8101	}
8102	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8103
8104	/*
8105	* !!! IMPORTANT !!!
8106	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8107	*/
8108
8109	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8110	/** @todo Deregistering here means we need to VMCLEAR always
8111	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8112	* for calling VMMR0ThreadCtxHookDisable here! */
8113	VMMR0ThreadCtxHookDisable(pVCpu);
8114
8115	/* Leave HM context. This takes care of local init (term). */
8116	int rc = HMR0LeaveCpu(pVCpu);
8117
8118	HM_RESTORE_PREEMPT();
8119	return rc;
8120	}
8121
8122
8123	/**
8124	* Does the necessary state syncing before doing a longjmp to ring-3.
8125	*
8126	* @returns VBox status code.
8127	* @param pVCpu The cross context virtual CPU structure.
8128	*
8129	* @remarks No-long-jmp zone!!!
8130	*/
8131	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8132	{
8133	return hmR0VmxLeaveSession(pVCpu);
8134	}
8135
8136
8137	/**
8138	* Take necessary actions before going back to ring-3.
8139	*
8140	* An action requires us to go back to ring-3. This function does the necessary
8141	* steps before we can safely return to ring-3. This is not the same as longjmps
8142	* to ring-3, this is voluntary and prepares the guest so it may continue
8143	* executing outside HM (recompiler/IEM).
8144	*
8145	* @returns VBox status code.
8146	* @param pVCpu The cross context virtual CPU structure.
8147	* @param rcExit The reason for exiting to ring-3. Can be
8148	* VINF_VMM_UNKNOWN_RING3_CALL.
8149	*/
8150	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8151	{
8152	Assert(pVCpu);
8153	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8154
8155	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8156	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8157	{
8158	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8159	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8160	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8161	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8162	}
8163
8164	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8165	VMMRZCallRing3Disable(pVCpu);
8166	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8167
8168	/*
8169	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8170	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8171	*
8172	* This is because execution may continue from ring-3 and we would need to inject
8173	* the event from there (hence place it back in TRPM).
8174	*/
8175	if (pVCpu->hm.s.Event.fPending)
8176	{
8177	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8178	Assert(!pVCpu->hm.s.Event.fPending);
8179
8180	/* Clear the events from the VMCS. */
8181	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8182	AssertRCReturn(rc, rc);
8183	}
8184	#ifdef VBOX_STRICT
8185	else
8186	{
8187	/*
8188	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8189	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8190	* occasionally, see @bugref{9180#c42}.
8191	*/
8192	uint32_t uEntryIntInfo;
8193	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8194	AssertRC(rc);
8195	Assert(!VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8196	}
8197	#endif
8198
8199	/*
8200	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8201	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8202	* (e.g. TPR below threshold).
8203	*/
8204	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8205	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8206	AssertRCReturn(rc, rc);
8207
8208	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8209	and if we're injecting an event we should have a TRPM trap pending. */
8210	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8211	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8212	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8213	#endif
8214
8215	/* Save guest state and restore host state bits. */
8216	rc = hmR0VmxLeaveSession(pVCpu);
8217	AssertRCReturn(rc, rc);
8218	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8219
8220	/* Thread-context hooks are unregistered at this point!!! */
8221
8222	/* Sync recompiler state. */
8223	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8224	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8225	\| CPUM_CHANGED_LDTR
8226	\| CPUM_CHANGED_GDTR
8227	\| CPUM_CHANGED_IDTR
8228	\| CPUM_CHANGED_TR
8229	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8230	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8231	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8232	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8233
8234	Assert(!pVCpu->hm.s.fClearTrapFlag);
8235
8236	/* Update the exit-to-ring 3 reason. */
8237	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8238
8239	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8240	if ( rcExit != VINF_EM_RAW_INTERRUPT
8241	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8242	{
8243	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8244	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8245	}
8246
8247	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8248
8249	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
8250	VMMRZCallRing3RemoveNotification(pVCpu);
8251	VMMRZCallRing3Enable(pVCpu);
8252
8253	return rc;
8254	}
8255
8256
8257	/**
8258	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8259	* longjump to ring-3 and possibly get preempted.
8260	*
8261	* @returns VBox status code.
8262	* @param pVCpu The cross context virtual CPU structure.
8263	* @param enmOperation The operation causing the ring-3 longjump.
8264	* @param pvUser User argument, currently unused, NULL.
8265	*/
8266	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
8267	{
8268	RT_NOREF(pvUser);
8269	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8270	{
8271	/*
8272	* !!! IMPORTANT !!!
8273	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8274	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8275	*/
8276	VMMRZCallRing3RemoveNotification(pVCpu);
8277	VMMRZCallRing3Disable(pVCpu);
8278	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
8279	RTThreadPreemptDisable(&PreemptState);
8280
8281	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8282	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8283	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8284	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8285
8286	#if HC_ARCH_BITS == 64
8287	/* Restore host-state bits that VT-x only restores partially. */
8288	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8289	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8290	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8291	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8292	#endif
8293
8294	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8295	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8296	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8297
8298	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8299	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8300	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8301
8302	/* Clear the current VMCS data back to memory. */
8303	hmR0VmxClearVmcs(pVmcsInfo);
8304
8305	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8306	VMMR0ThreadCtxHookDisable(pVCpu);
8307	HMR0LeaveCpu(pVCpu);
8308	RTThreadPreemptRestore(&PreemptState);
8309	return VINF_SUCCESS;
8310	}
8311
8312	Assert(pVCpu);
8313	Assert(pvUser);
8314	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8315	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8316
8317	VMMRZCallRing3Disable(pVCpu);
8318	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8319
8320	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8321
8322	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8323	AssertRCReturn(rc, rc);
8324
8325	VMMRZCallRing3Enable(pVCpu);
8326	return VINF_SUCCESS;
8327	}
8328
8329
8330	/**
8331	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8332	* stack.
8333	*
8334	* @returns Strict VBox status code (i.e. informational status codes too).
8335	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8336	* @param pVCpu The cross context virtual CPU structure.
8337	* @param uValue The value to push to the guest stack.
8338	*/
8339	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
8340	{
8341	/*
8342	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8343	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8344	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8345	*/
8346	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8347	if (pCtx->sp == 1)
8348	return VINF_EM_RESET;
8349	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8350	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8351	AssertRC(rc);
8352	return rc;
8353	}
8354
8355
8356	/**
8357	* Injects an event into the guest upon VM-entry by updating the relevant fields
8358	* in the VM-entry area in the VMCS.
8359	*
8360	* @returns Strict VBox status code (i.e. informational status codes too).
8361	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8362	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8363	*
8364	* @param pVCpu The cross context virtual CPU structure.
8365	* @param pVmxTransient The VMX-transient structure.
8366	* @param pEvent The event being injected.
8367	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state.
8368	* This will be updated if necessary. This cannot not
8369	* be NULL.
8370	* @param fStepping Whether we're single-stepping guest execution and
8371	* should return VINF_EM_DBG_STEPPED if the event is
8372	* injected directly (registers modified by us, not by
8373	* hardware on VM-entry).
8374	*/
8375	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8376	uint32_t *pfIntrState)
8377	{
8378	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8379	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8380	Assert(pfIntrState);
8381
8382	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8383	uint32_t u32IntInfo = pEvent->u64IntInfo;
8384	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8385	uint32_t const cbInstr = pEvent->cbInstr;
8386	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8387	uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8388	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8389
8390	#ifdef VBOX_STRICT
8391	/*
8392	* Validate the error-code-valid bit for hardware exceptions.
8393	* No error codes for exceptions in real-mode.
8394	*
8395	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8396	*/
8397	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8398	&& !CPUMIsGuestInRealModeEx(pCtx))
8399	{
8400	switch (uVector)
8401	{
8402	case X86_XCPT_PF:
8403	case X86_XCPT_DF:
8404	case X86_XCPT_TS:
8405	case X86_XCPT_NP:
8406	case X86_XCPT_SS:
8407	case X86_XCPT_GP:
8408	case X86_XCPT_AC:
8409	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8410	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8411	RT_FALL_THRU();
8412	default:
8413	break;
8414	}
8415	}
8416
8417	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8418	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8419	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8420	#endif
8421
8422	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8423
8424	/*
8425	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8426	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8427	* interrupt handler in the (real-mode) guest.
8428	*
8429	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8430	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8431	*/
8432	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8433	{
8434	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8435	{
8436	/*
8437	* For CPUs with unrestricted guest execution enabled and with the guest
8438	* in real-mode, we must not set the deliver-error-code bit.
8439	*
8440	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8441	*/
8442	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8443	}
8444	else
8445	{
8446	PVM pVM = pVCpu->CTX_SUFF(pVM);
8447	Assert(PDMVmmDevHeapIsEnabled(pVM));
8448	Assert(pVM->hm.s.vmx.pRealModeTSS);
8449	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8450
8451	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8452	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8453	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8454	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8455	AssertRCReturn(rc2, rc2);
8456
8457	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8458	size_t const cbIdtEntry = sizeof(X86IDTR16);
8459	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8460	{
8461	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8462	if (uVector == X86_XCPT_DF)
8463	return VINF_EM_RESET;
8464
8465	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8466	No error codes for exceptions in real-mode. */
8467	if (uVector == X86_XCPT_GP)
8468	{
8469	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8470	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8471	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8472	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8473	HMEVENT EventXcptDf;
8474	RT_ZERO(EventXcptDf);
8475	EventXcptDf.u64IntInfo = uXcptDfInfo;
8476	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8477	}
8478
8479	/*
8480	* If we're injecting an event with no valid IDT entry, inject a #GP.
8481	* No error codes for exceptions in real-mode.
8482	*
8483	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8484	*/
8485	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8486	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8487	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8488	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8489	HMEVENT EventXcptGp;
8490	RT_ZERO(EventXcptGp);
8491	EventXcptGp.u64IntInfo = uXcptGpInfo;
8492	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8493	}
8494
8495	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8496	uint16_t uGuestIp = pCtx->ip;
8497	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8498	{
8499	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8500	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8501	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8502	}
8503	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8504	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8505
8506	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8507	X86IDTR16 IdtEntry;
8508	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8509	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8510	AssertRCReturn(rc2, rc2);
8511
8512	/* Construct the stack frame for the interrupt/exception handler. */
8513	VBOXSTRICTRC rcStrict;
8514	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8515	if (rcStrict == VINF_SUCCESS)
8516	{
8517	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8518	if (rcStrict == VINF_SUCCESS)
8519	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8520	}
8521
8522	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8523	if (rcStrict == VINF_SUCCESS)
8524	{
8525	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8526	pCtx->rip = IdtEntry.offSel;
8527	pCtx->cs.Sel = IdtEntry.uSel;
8528	pCtx->cs.ValidSel = IdtEntry.uSel;
8529	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8530	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8531	&& uVector == X86_XCPT_PF)
8532	pCtx->cr2 = GCPtrFault;
8533
8534	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8535	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8536	\| HM_CHANGED_GUEST_RSP);
8537
8538	/*
8539	* If we delivered a hardware exception (other than an NMI) and if there was
8540	* block-by-STI in effect, we should clear it.
8541	*/
8542	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8543	{
8544	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8545	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8546	Log4Func(("Clearing inhibition due to STI\n"));
8547	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8548	}
8549
8550	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8551	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8552
8553	/*
8554	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8555	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8556	*/
8557	pVCpu->hm.s.Event.fPending = false;
8558
8559	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8560	if (fStepping)
8561	rcStrict = VINF_EM_DBG_STEPPED;
8562	}
8563	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8564	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8565	return rcStrict;
8566	}
8567	}
8568
8569	/*
8570	* Validate.
8571	*/
8572	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8573	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8574
8575	/*
8576	* Inject the event into the VMCS.
8577	*/
8578	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8579	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8580	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8581	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8582	AssertRCReturn(rc, rc);
8583
8584	/*
8585	* Update guest CR2 if this is a page-fault.
8586	*/
8587	if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8588	&& uVector == X86_XCPT_PF)
8589	pCtx->cr2 = GCPtrFault;
8590
8591	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8592	return VINF_SUCCESS;
8593	}
8594
8595
8596	/**
8597	* Evaluates the event to be delivered to the guest and sets it as the pending
8598	* event.
8599	*
8600	* @returns Strict VBox status code (i.e. informational status codes too).
8601	* @param pVCpu The cross context virtual CPU structure.
8602	* @param pVmxTransient The VMX-transient structure.
8603	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8604	*/
8605	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8606	{
8607	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8608	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8609
8610	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
8611	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmcsInfo);
8612	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8613	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8614	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8615
8616	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8617	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8618	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8619	Assert(!TRPMHasTrap(pVCpu));
8620	Assert(pfIntrState);
8621
8622	*pfIntrState = fIntrState;
8623
8624	/*
8625	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
8626	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
8627	*/
8628	/** @todo SMI. SMIs take priority over NMIs. */
8629	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
8630	{
8631	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8632	if ( !pVCpu->hm.s.Event.fPending
8633	&& !fBlockNmi
8634	&& !fBlockSti
8635	&& !fBlockMovSS)
8636	{
8637	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8638	if ( pVmxTransient->fIsNestedGuest
8639	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8640	return IEMExecVmxVmexitNmi(pVCpu);
8641	#endif
8642	hmR0VmxSetPendingXcptNmi(pVCpu);
8643	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8644	Log4Func(("Pending NMI\n"));
8645	}
8646	else
8647	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8648	}
8649	/*
8650	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
8651	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
8652	*/
8653	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8654	&& !pVCpu->hm.s.fSingleInstruction)
8655	{
8656	Assert(!DBGFIsStepping(pVCpu));
8657	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8658	AssertRCReturn(rc, rc);
8659	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8660	if ( !pVCpu->hm.s.Event.fPending
8661	&& !fBlockInt
8662	&& !fBlockSti
8663	&& !fBlockMovSS)
8664	{
8665	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8666	if ( pVmxTransient->fIsNestedGuest
8667	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
8668	{
8669	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector /, true / fIntPending */);
8670	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8671	return rcStrict;
8672	}
8673	#endif
8674	uint8_t u8Interrupt;
8675	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8676	if (RT_SUCCESS(rc))
8677	{
8678	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8679	if ( pVmxTransient->fIsNestedGuest
8680	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8681	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8682	{
8683	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8684	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8685	return rcStrict;
8686	}
8687	#endif
8688	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8689	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
8690	}
8691	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8692	{
8693	if ( !pVmxTransient->fIsNestedGuest
8694	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8695	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
8696	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8697
8698	/*
8699	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8700	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8701	* need to re-set this force-flag here.
8702	*/
8703	}
8704	else
8705	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8706	}
8707	else
8708	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8709	}
8710
8711	return VINF_SUCCESS;
8712	}
8713
8714
8715	/**
8716	* Injects any pending events into the guest if the guest is in a state to
8717	* receive them.
8718	*
8719	* @returns Strict VBox status code (i.e. informational status codes too).
8720	* @param pVCpu The cross context virtual CPU structure.
8721	* @param pVmxTransient The VMX-transient structure.
8722	* @param fIntrState The VT-x guest-interruptibility state.
8723	* @param fStepping Whether we are single-stepping the guest using the
8724	* hypervisor debugger and should return
8725	* VINF_EM_DBG_STEPPED if the event was dispatched
8726	* directly.
8727	*/
8728	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
8729	{
8730	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8731	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8732
8733	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8734	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8735
8736	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8737	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8738	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8739	Assert(!TRPMHasTrap(pVCpu));
8740
8741	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8742	if (pVCpu->hm.s.Event.fPending)
8743	{
8744	/*
8745	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8746	* pending even while injecting an event and in this case, we want a VM-exit as soon as
8747	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
8748	*
8749	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8750	*/
8751	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8752	#ifdef VBOX_STRICT
8753	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8754	{
8755	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
8756	Assert(!fBlockInt);
8757	Assert(!fBlockSti);
8758	Assert(!fBlockMovSS);
8759	}
8760	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
8761	{
8762	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8763	Assert(!fBlockSti);
8764	Assert(!fBlockMovSS);
8765	Assert(!fBlockNmi);
8766	}
8767	#endif
8768	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8769	uIntType));
8770
8771	/*
8772	* Inject the event and get any changes to the guest-interruptibility state.
8773	*
8774	* The guest-interruptibility state may need to be updated if we inject the event
8775	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
8776	*/
8777	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
8778	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8779
8780	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8781	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8782	else
8783	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8784	}
8785
8786	/*
8787	* Update the guest-interruptibility state.
8788	*
8789	* This is required for the real-on-v86 software interrupt injection case above, as well as
8790	* updates to the guest state from ring-3 or IEM/REM.
8791	*/
8792	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
8793	AssertRCReturn(rc, rc);
8794
8795	/*
8796	* There's no need to clear the VM-entry interruption-information field here if we're not
8797	* injecting anything. VT-x clears the valid bit on every VM-exit.
8798	*
8799	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8800	*/
8801
8802	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8803	NOREF(fBlockMovSS); NOREF(fBlockSti);
8804	return rcStrict;
8805	}
8806
8807
8808	/**
8809	* Enters the VT-x session.
8810	*
8811	* @returns VBox status code.
8812	* @param pVCpu The cross context virtual CPU structure.
8813	*/
8814	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
8815	{
8816	AssertPtr(pVCpu);
8817	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8818	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8819
8820	LogFlowFunc(("pVCpu=%p\n", pVCpu));
8821	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8822	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8823
8824	#ifdef VBOX_STRICT
8825	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8826	RTCCUINTREG uHostCR4 = ASMGetCR4();
8827	if (!(uHostCR4 & X86_CR4_VMXE))
8828	{
8829	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8830	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8831	}
8832	#endif
8833
8834	/*
8835	* Load the appropriate VMCS as the current and active one.
8836	*/
8837	PVMXVMCSINFO pVmcsInfo;
8838	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
8839	if (!fInNestedGuestMode)
8840	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
8841	else
8842	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
8843	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
8844	if (RT_SUCCESS(rc))
8845	{
8846	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
8847	pVCpu->hm.s.fLeaveDone = false;
8848	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8849
8850	/*
8851	* Do the EMT scheduled L1D flush here if needed.
8852	*/
8853	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8854	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8855	}
8856	return rc;
8857	}
8858
8859
8860	/**
8861	* The thread-context callback (only on platforms which support it).
8862	*
8863	* @param enmEvent The thread-context event.
8864	* @param pVCpu The cross context virtual CPU structure.
8865	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8866	* @thread EMT(pVCpu)
8867	*/
8868	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8869	{
8870	NOREF(fGlobalInit);
8871
8872	switch (enmEvent)
8873	{
8874	case RTTHREADCTXEVENT_OUT:
8875	{
8876	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8877	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8878	VMCPU_ASSERT_EMT(pVCpu);
8879
8880	/* No longjmps (logger flushes, locks) in this fragile context. */
8881	VMMRZCallRing3Disable(pVCpu);
8882	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8883
8884	/* Restore host-state (FPU, debug etc.) */
8885	if (!pVCpu->hm.s.fLeaveDone)
8886	{
8887	/*
8888	* Do -not- import the guest-state here as we might already be in the middle of importing
8889	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8890	*/
8891	hmR0VmxLeave(pVCpu, false /* fImportState */);
8892	pVCpu->hm.s.fLeaveDone = true;
8893	}
8894
8895	/* Leave HM context, takes care of local init (term). */
8896	int rc = HMR0LeaveCpu(pVCpu);
8897	AssertRC(rc);
8898
8899	/* Restore longjmp state. */
8900	VMMRZCallRing3Enable(pVCpu);
8901	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8902	break;
8903	}
8904
8905	case RTTHREADCTXEVENT_IN:
8906	{
8907	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8908	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8909	VMCPU_ASSERT_EMT(pVCpu);
8910
8911	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8912	VMMRZCallRing3Disable(pVCpu);
8913	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8914
8915	/* Initialize the bare minimum state required for HM. This takes care of
8916	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8917	int rc = hmR0EnterCpu(pVCpu);
8918	AssertRC(rc);
8919	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8920	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8921
8922	/* Load the active VMCS as the current one. */
8923	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8924	rc = hmR0VmxLoadVmcs(pVmcsInfo);
8925	AssertRC(rc);
8926	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8927	pVCpu->hm.s.fLeaveDone = false;
8928
8929	/* Do the EMT scheduled L1D flush if needed. */
8930	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8931	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8932
8933	/* Restore longjmp state. */
8934	VMMRZCallRing3Enable(pVCpu);
8935	break;
8936	}
8937
8938	default:
8939	break;
8940	}
8941	}
8942
8943
8944	/**
8945	* Exports the host state into the VMCS host-state area.
8946	* Sets up the VM-exit MSR-load area.
8947	*
8948	* The CPU state will be loaded from these fields on every successful VM-exit.
8949	*
8950	* @returns VBox status code.
8951	* @param pVCpu The cross context virtual CPU structure.
8952	*
8953	* @remarks No-long-jump zone!!!
8954	*/
8955	static int hmR0VmxExportHostState(PVMCPU pVCpu)
8956	{
8957	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8958
8959	int rc = VINF_SUCCESS;
8960	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8961	{
8962	rc = hmR0VmxExportHostControlRegs();
8963	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8964
8965	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8966	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8967
8968	rc = hmR0VmxExportHostMsrs(pVCpu);
8969	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8970
8971	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8972	}
8973	return rc;
8974	}
8975
8976
8977	/**
8978	* Saves the host state in the VMCS host-state.
8979	*
8980	* @returns VBox status code.
8981	* @param pVCpu The cross context virtual CPU structure.
8982	*
8983	* @remarks No-long-jump zone!!!
8984	*/
8985	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
8986	{
8987	AssertPtr(pVCpu);
8988	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8989
8990	/*
8991	* Export the host state here while entering HM context.
8992	* When thread-context hooks are used, we might get preempted and have to re-save the host
8993	* state but most of the time we won't be, so do it here before we disable interrupts.
8994	*/
8995	return hmR0VmxExportHostState(pVCpu);
8996	}
8997
8998
8999	/**
9000	* Exports the guest state into the VMCS guest-state area.
9001	*
9002	* The will typically be done before VM-entry when the guest-CPU state and the
9003	* VMCS state may potentially be out of sync.
9004	*
9005	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9006	* VM-entry controls.
9007	* Sets up the appropriate VMX non-root function to execute guest code based on
9008	* the guest CPU mode.
9009	*
9010	* @returns VBox strict status code.
9011	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9012	* without unrestricted guest execution and the VMMDev is not presently
9013	* mapped (e.g. EFI32).
9014	*
9015	* @param pVCpu The cross context virtual CPU structure.
9016	* @param pVmxTransient The VMX-transient structure.
9017	*
9018	* @remarks No-long-jump zone!!!
9019	*/
9020	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9021	{
9022	AssertPtr(pVCpu);
9023	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9024	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9025
9026	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9027
9028	/*
9029	* Determine real-on-v86 mode.
9030	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9031	*/
9032	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9033	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9034	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9035	pVmcsInfo->RealMode. fRealOnV86Active = false;
9036	else
9037	{
9038	Assert(!pVmxTransient->fIsNestedGuest);
9039	pVmcsInfo->RealMode.fRealOnV86Active = true;
9040	}
9041
9042	/*
9043	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9044	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9045	*/
9046	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9047	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9048	* be a need to evaluate this everytime since I'm pretty sure we intercept
9049	* all guest paging mode changes. */
9050	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9051	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9052
9053	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9054	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9055
9056	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9057	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9058
9059	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9060	if (rcStrict == VINF_SUCCESS)
9061	{ /* likely */ }
9062	else
9063	{
9064	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9065	return rcStrict;
9066	}
9067
9068	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9069	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9070
9071	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9072	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9073
9074	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9075	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9076
9077	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9078	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9079
9080	rc = hmR0VmxExportGuestRip(pVCpu);
9081	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9082	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9083	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9084
9085	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9086	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9087	\| HM_CHANGED_GUEST_CR2
9088	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9089	\| HM_CHANGED_GUEST_X87
9090	\| HM_CHANGED_GUEST_SSE_AVX
9091	\| HM_CHANGED_GUEST_OTHER_XSAVE
9092	\| HM_CHANGED_GUEST_XCRx
9093	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9094	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9095	\| HM_CHANGED_GUEST_TSC_AUX
9096	\| HM_CHANGED_GUEST_OTHER_MSRS
9097	\| HM_CHANGED_GUEST_HWVIRT /* More accurate PLE handling someday? */
9098	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9099
9100	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9101	return rc;
9102	}
9103
9104
9105	/**
9106	* Exports the state shared between the host and guest into the VMCS.
9107	*
9108	* @param pVCpu The cross context virtual CPU structure.
9109	* @param pVmxTransient The VMX-transient structure.
9110	*
9111	* @remarks No-long-jump zone!!!
9112	*/
9113	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9114	{
9115	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9116	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9117
9118	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9119	{
9120	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9121	AssertRC(rc);
9122	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9123
9124	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9125	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9126	{
9127	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9128	AssertRC(rc);
9129	}
9130	}
9131
9132	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9133	{
9134	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9135	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9136	}
9137
9138	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9139	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9140	}
9141
9142
9143	/**
9144	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9145	*
9146	* @returns Strict VBox status code (i.e. informational status codes too).
9147	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9148	* without unrestricted guest execution and the VMMDev is not presently
9149	* mapped (e.g. EFI32).
9150	*
9151	* @param pVCpu The cross context virtual CPU structure.
9152	* @param pVmxTransient The VMX-transient structure.
9153	*
9154	* @remarks No-long-jump zone!!!
9155	*/
9156	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9157	{
9158	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9159	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9160	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9161
9162	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9163	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9164	#endif
9165
9166	/*
9167	* For many exits it's only RIP that changes and hence try to export it first
9168	* without going through a lot of change flag checks.
9169	*/
9170	VBOXSTRICTRC rcStrict;
9171	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9172	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9173	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9174	{
9175	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9176	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9177	{ /* likely */}
9178	else
9179	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9180	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9181	}
9182	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9183	{
9184	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9185	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9186	{ /* likely */}
9187	else
9188	{
9189	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9190	VBOXSTRICTRC_VAL(rcStrict)));
9191	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9192	return rcStrict;
9193	}
9194	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9195	}
9196	else
9197	rcStrict = VINF_SUCCESS;
9198
9199	#ifdef VBOX_STRICT
9200	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9201	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9202	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9203	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9204	("fCtxChanged=%#RX64\n", fCtxChanged));
9205	#endif
9206	return rcStrict;
9207	}
9208
9209
9210	/**
9211	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9212	* and update error record fields accordingly.
9213	*
9214	* @return VMX_IGS_* return codes.
9215	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9216	* wrong with the guest state.
9217	*
9218	* @param pVCpu The cross context virtual CPU structure.
9219	* @param pVmcsInfo The VMCS info. object.
9220	*
9221	* @remarks This function assumes our cache of the VMCS controls
9222	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9223	*/
9224	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
9225	{
9226	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9227	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9228	uError = (err); \
9229	break; \
9230	} else do { } while (0)
9231
9232	int rc;
9233	PVM pVM = pVCpu->CTX_SUFF(pVM);
9234	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9235	uint32_t uError = VMX_IGS_ERROR;
9236	uint32_t u32Val;
9237	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9238
9239	do
9240	{
9241	/*
9242	* CR0.
9243	*/
9244	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9245	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9246	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
9247	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9248	if (fUnrestrictedGuest)
9249	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
9250
9251	uint32_t u32GuestCr0;
9252	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
9253	AssertRCBreak(rc);
9254	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9255	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9256	if ( !fUnrestrictedGuest
9257	&& (u32GuestCr0 & X86_CR0_PG)
9258	&& !(u32GuestCr0 & X86_CR0_PE))
9259	{
9260	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9261	}
9262
9263	/*
9264	* CR4.
9265	*/
9266	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9267	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9268
9269	uint32_t u32GuestCr4;
9270	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
9271	AssertRCBreak(rc);
9272	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9273	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9274
9275	/*
9276	* IA32_DEBUGCTL MSR.
9277	*/
9278	uint64_t u64Val;
9279	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9280	AssertRCBreak(rc);
9281	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9282	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9283	{
9284	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9285	}
9286	uint64_t u64DebugCtlMsr = u64Val;
9287
9288	#ifdef VBOX_STRICT
9289	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9290	AssertRCBreak(rc);
9291	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9292	#endif
9293	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9294
9295	/*
9296	* RIP and RFLAGS.
9297	*/
9298	uint32_t u32Eflags;
9299	#if HC_ARCH_BITS == 64
9300	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9301	AssertRCBreak(rc);
9302	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9303	if ( !fLongModeGuest
9304	\|\| !pCtx->cs.Attr.n.u1Long)
9305	{
9306	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9307	}
9308	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9309	* must be identical if the "IA-32e mode guest" VM-entry
9310	* control is 1 and CS.L is 1. No check applies if the
9311	* CPU supports 64 linear-address bits. */
9312
9313	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9314	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9315	AssertRCBreak(rc);
9316	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9317	VMX_IGS_RFLAGS_RESERVED);
9318	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9319	u32Eflags = u64Val;
9320	#else
9321	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9322	AssertRCBreak(rc);
9323	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9324	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9325	#endif
9326
9327	if ( fLongModeGuest
9328	\|\| ( fUnrestrictedGuest
9329	&& !(u32GuestCr0 & X86_CR0_PE)))
9330	{
9331	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9332	}
9333
9334	uint32_t u32EntryInfo;
9335	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9336	AssertRCBreak(rc);
9337	if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9338	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9339	{
9340	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9341	}
9342
9343	/*
9344	* 64-bit checks.
9345	*/
9346	#if HC_ARCH_BITS == 64
9347	if (fLongModeGuest)
9348	{
9349	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9350	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9351	}
9352
9353	if ( !fLongModeGuest
9354	&& (u32GuestCr4 & X86_CR4_PCIDE))
9355	{
9356	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9357	}
9358
9359	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9360	* 51:32 beyond the processor's physical-address width are 0. */
9361
9362	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9363	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9364	{
9365	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9366	}
9367
9368	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9369	AssertRCBreak(rc);
9370	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9371
9372	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9373	AssertRCBreak(rc);
9374	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9375	#endif
9376
9377	/*
9378	* PERF_GLOBAL MSR.
9379	*/
9380	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9381	{
9382	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9383	AssertRCBreak(rc);
9384	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9385	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9386	}
9387
9388	/*
9389	* PAT MSR.
9390	*/
9391	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9392	{
9393	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9394	AssertRCBreak(rc);
9395	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9396	for (unsigned i = 0; i < 8; i++)
9397	{
9398	uint8_t u8Val = (u64Val & 0xff);
9399	if ( u8Val != 0 /* UC */
9400	&& u8Val != 1 /* WC */
9401	&& u8Val != 4 /* WT */
9402	&& u8Val != 5 /* WP */
9403	&& u8Val != 6 /* WB */
9404	&& u8Val != 7 /* UC- */)
9405	{
9406	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9407	}
9408	u64Val >>= 8;
9409	}
9410	}
9411
9412	/*
9413	* EFER MSR.
9414	*/
9415	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9416	{
9417	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9418	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9419	AssertRCBreak(rc);
9420	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9421	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9422	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9423	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9424	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9425	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9426	* iemVmxVmentryCheckGuestState(). */
9427	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9428	\|\| !(u32GuestCr0 & X86_CR0_PG)
9429	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9430	VMX_IGS_EFER_LMA_LME_MISMATCH);
9431	}
9432
9433	/*
9434	* Segment registers.
9435	*/
9436	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9437	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9438	if (!(u32Eflags & X86_EFL_VM))
9439	{
9440	/* CS */
9441	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9442	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9443	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9444	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9445	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9446	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9447	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9448	/* CS cannot be loaded with NULL in protected mode. */
9449	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9450	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9451	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9452	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9453	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9454	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9455	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9456	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9457	else
9458	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9459
9460	/* SS */
9461	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9462	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9463	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9464	if ( !(pCtx->cr0 & X86_CR0_PE)
9465	\|\| pCtx->cs.Attr.n.u4Type == 3)
9466	{
9467	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9468	}
9469	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9470	{
9471	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9472	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9473	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9474	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9475	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9476	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9477	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9478	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9479	}
9480
9481	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9482	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9483	{
9484	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9485	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9486	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9487	\|\| pCtx->ds.Attr.n.u4Type > 11
9488	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9489	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9490	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9491	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9492	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9493	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9494	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9495	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9496	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9497	}
9498	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9499	{
9500	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9501	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9502	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9503	\|\| pCtx->es.Attr.n.u4Type > 11
9504	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9505	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9506	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9507	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9508	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9509	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9510	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9511	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9512	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9513	}
9514	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9515	{
9516	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9517	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9518	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9519	\|\| pCtx->fs.Attr.n.u4Type > 11
9520	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9521	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9522	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9523	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9524	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9525	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9526	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9527	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9528	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9529	}
9530	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9531	{
9532	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9533	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9534	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9535	\|\| pCtx->gs.Attr.n.u4Type > 11
9536	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9537	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9538	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9539	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9540	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9541	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9542	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9543	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9544	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9545	}
9546	/* 64-bit capable CPUs. */
9547	#if HC_ARCH_BITS == 64
9548	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9549	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9550	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9551	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9552	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9553	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9554	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9555	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9556	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9557	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9558	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9559	#endif
9560	}
9561	else
9562	{
9563	/* V86 mode checks. */
9564	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9565	if (pVmcsInfo->RealMode.fRealOnV86Active)
9566	{
9567	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9568	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9569	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9570	}
9571	else
9572	{
9573	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9574	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9575	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9576	}
9577
9578	/* CS */
9579	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9580	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9581	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9582	/* SS */
9583	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9584	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9585	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9586	/* DS */
9587	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9588	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9589	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9590	/* ES */
9591	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9592	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9593	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9594	/* FS */
9595	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9596	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9597	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9598	/* GS */
9599	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9600	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9601	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9602	/* 64-bit capable CPUs. */
9603	#if HC_ARCH_BITS == 64
9604	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9605	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9606	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9607	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9608	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9609	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9610	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9611	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9612	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9613	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9614	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9615	#endif
9616	}
9617
9618	/*
9619	* TR.
9620	*/
9621	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9622	/* 64-bit capable CPUs. */
9623	#if HC_ARCH_BITS == 64
9624	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9625	#endif
9626	if (fLongModeGuest)
9627	{
9628	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9629	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9630	}
9631	else
9632	{
9633	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9634	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9635	VMX_IGS_TR_ATTR_TYPE_INVALID);
9636	}
9637	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9638	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9639	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9640	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9641	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9642	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9643	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9644	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9645
9646	/*
9647	* GDTR and IDTR.
9648	*/
9649	#if HC_ARCH_BITS == 64
9650	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9651	AssertRCBreak(rc);
9652	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9653
9654	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9655	AssertRCBreak(rc);
9656	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9657	#endif
9658
9659	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9660	AssertRCBreak(rc);
9661	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9662
9663	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9664	AssertRCBreak(rc);
9665	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9666
9667	/*
9668	* Guest Non-Register State.
9669	*/
9670	/* Activity State. */
9671	uint32_t u32ActivityState;
9672	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9673	AssertRCBreak(rc);
9674	HMVMX_CHECK_BREAK( !u32ActivityState
9675	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9676	VMX_IGS_ACTIVITY_STATE_INVALID);
9677	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9678	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9679	uint32_t u32IntrState;
9680	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9681	AssertRCBreak(rc);
9682	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9683	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9684	{
9685	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9686	}
9687
9688	/** @todo Activity state and injecting interrupts. Left as a todo since we
9689	* currently don't use activity states but ACTIVE. */
9690
9691	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9692	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9693
9694	/* Guest interruptibility-state. */
9695	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9696	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9697	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9698	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9699	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9700	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9701	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9702	if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
9703	{
9704	if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9705	{
9706	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9707	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9708	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9709	}
9710	else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9711	{
9712	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9713	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9714	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9715	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9716	}
9717	}
9718	/** @todo Assumes the processor is not in SMM. */
9719	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9720	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9721	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9722	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9723	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9724	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9725	&& VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9726	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9727	{
9728	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
9729	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9730	}
9731
9732	/* Pending debug exceptions. */
9733	#if HC_ARCH_BITS == 64
9734	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
9735	AssertRCBreak(rc);
9736	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9737	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9738	u32Val = u64Val; /* For pending debug exceptions checks below. */
9739	#else
9740	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
9741	AssertRCBreak(rc);
9742	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9743	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9744	#endif
9745
9746	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9747	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
9748	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9749	{
9750	if ( (u32Eflags & X86_EFL_TF)
9751	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9752	{
9753	/* Bit 14 is PendingDebug.BS. */
9754	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9755	}
9756	if ( !(u32Eflags & X86_EFL_TF)
9757	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9758	{
9759	/* Bit 14 is PendingDebug.BS. */
9760	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9761	}
9762	}
9763
9764	/* VMCS link pointer. */
9765	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9766	AssertRCBreak(rc);
9767	if (u64Val != UINT64_C(0xffffffffffffffff))
9768	{
9769	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9770	/** @todo Bits beyond the processor's physical-address width MBZ. */
9771	/** @todo 32-bit located in memory referenced by value of this field (as a
9772	* physical address) must contain the processor's VMCS revision ID. */
9773	/** @todo SMM checks. */
9774	}
9775
9776	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9777	* not using nested paging? */
9778	if ( pVM->hm.s.fNestedPaging
9779	&& !fLongModeGuest
9780	&& CPUMIsGuestInPAEModeEx(pCtx))
9781	{
9782	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9783	AssertRCBreak(rc);
9784	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9785
9786	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9787	AssertRCBreak(rc);
9788	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9789
9790	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9791	AssertRCBreak(rc);
9792	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9793
9794	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9795	AssertRCBreak(rc);
9796	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9797	}
9798
9799	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9800	if (uError == VMX_IGS_ERROR)
9801	uError = VMX_IGS_REASON_NOT_FOUND;
9802	} while (0);
9803
9804	pVCpu->hm.s.u32HMError = uError;
9805	return uError;
9806
9807	#undef HMVMX_ERROR_BREAK
9808	#undef HMVMX_CHECK_BREAK
9809	}
9810
9811
9812	/**
9813	* Setup the APIC-access page for virtualizing APIC access.
9814	*
9815	* This can cause a longjumps to R3 due to the acquisition of the PGM lock, hence
9816	* this not done as part of exporting guest state, see @bugref{8721}.
9817	*
9818	* @returns VBox status code.
9819	* @param pVCpu The cross context virtual CPU structure.
9820	*/
9821	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
9822	{
9823	PVM pVM = pVCpu->CTX_SUFF(pVM);
9824	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
9825
9826	Assert(PDMHasApic(pVM));
9827	Assert(u64MsrApicBase);
9828
9829	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
9830	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
9831
9832	/* Unalias any existing mapping. */
9833	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
9834	AssertRCReturn(rc, rc);
9835
9836	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
9837	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
9838	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
9839	AssertRCReturn(rc, rc);
9840
9841	/* Update the per-VCPU cache of the APIC base MSR. */
9842	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
9843	return VINF_SUCCESS;
9844	}
9845
9846
9847	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9848	/**
9849	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
9850	* nested-guest using hardware-assisted VMX.
9851	*
9852	* @param pVCpu The cross context virtual CPU structure.
9853	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
9854	* @param pVmcsInfoGst The guest VMCS info. object.
9855	*/
9856	static void hmR0VmxMergeMsrBitmapNested(PCVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
9857	{
9858	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
9859	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
9860	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
9861	Assert(pu64MsrBitmapNstGst);
9862	Assert(pu64MsrBitmapGst);
9863	Assert(pu64MsrBitmap);
9864
9865	/*
9866	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
9867	* MSR that is intercepted by the guest is also intercepted while executing the
9868	* nested-guest using hardware-assisted VMX.
9869	*/
9870	uint32_t const cbFrag = sizeof(uint64_t);
9871	uint32_t const cFrags = X86_PAGE_4K_SIZE / cbFrag;
9872	for (uint32_t i = 0; i <= cFrags; i++)
9873	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
9874	}
9875
9876
9877	/**
9878	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
9879	* hardware-assisted VMX execution of the nested-guest.
9880	*
9881	* For a guest, we don't modify these controls once we set up the VMCS.
9882	*
9883	* For nested-guests since the guest hypervisor provides these controls on every
9884	* nested-guest VM-entry and could potentially change them everytime we need to
9885	* merge them before every nested-guest VM-entry.
9886	*
9887	* @returns VBox status code.
9888	* @param pVCpu The cross context virtual CPU structure.
9889	*/
9890	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
9891	{
9892	PVM pVM = pVCpu->CTX_SUFF(pVM);
9893	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
9894	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
9895	Assert(pVmcsNstGst);
9896
9897	/*
9898	* Merge the controls with the requirements of the guest VMCS.
9899	*
9900	* We do not need to validate the nested-guest VMX features specified in the
9901	* nested-guest VMCS with the features supported by the physical CPU as it's
9902	* already done by the VMLAUNCH/VMRESUME instruction emulation.
9903	*
9904	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the
9905	* guest are derived from the VMX features supported by the physical CPU.
9906	*/
9907
9908	/* Pin-based VM-execution controls. */
9909	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
9910
9911	/* Processor-based VM-execution controls. */
9912	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
9913	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
9914	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
9915	\| VMX_PROC_CTLS_USE_TPR_SHADOW
9916	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
9917
9918	/* Secondary processor-based VM-execution controls. */
9919	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
9920	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
9921	\| VMX_PROC_CTLS2_INVPCID
9922	\| VMX_PROC_CTLS2_RDTSCP
9923	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
9924	\| VMX_PROC_CTLS2_APIC_REG_VIRT
9925	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
9926	\| VMX_PROC_CTLS2_VMFUNC));
9927
9928	/*
9929	* VM-entry controls:
9930	* These controls contains state that depends on the nested-guest state (primarily
9931	* EFER MSR) and is thus not constant through VMLAUNCH/VMRESUME and the nested-guest
9932	* VM-exit. Although the nested-hypervisor cannot change it, we need to in order to
9933	* properly continue executing the nested-guest if the EFER MSR changes but does not
9934	* cause a nested-guest VM-exits.
9935	*
9936	* VM-exit controls:
9937	* These controls specify the host state on return. We cannot use the controls from
9938	* the nested-hypervisor state as is as it would contain the guest state rather than
9939	* the host state. Since the host state is subject to change (e.g. preemption, trips
9940	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
9941	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
9942	*
9943	* VM-entry MSR-load:
9944	* The guest MSRs from the VM-entry MSR-load area are already loaded into the
9945	* guest-CPU context by the VMLAUNCH/VMRESUME instruction emulation.
9946	*
9947	* VM-exit MSR-store:
9948	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU
9949	* context back into the VM-exit MSR-store area.
9950	*
9951	* VM-exit MSR-load areas:
9952	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence,
9953	* we can entirely ignore what the nested-hypervisor wants to load here.
9954	*/
9955
9956	/*
9957	* Exception bitmap.
9958	*
9959	* We could remove #UD from the guest bitmap and merge it with the nested-guest
9960	* bitmap here (and avoid doing anything while exporting nested-guest state), but to
9961	* keep the code more flexible if intercepting exceptions become more dynamic in
9962	* the future we do it as part of exporting the nested-guest state.
9963	*/
9964	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
9965
9966	/*
9967	* CR0/CR4 guest/host mask.
9968	*
9969	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest
9970	* must cause VM-exits, so we need to merge them here.
9971	*/
9972	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
9973	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
9974
9975	/*
9976	* Page-fault error-code mask and match.
9977	*
9978	* Although we require unrestricted guest execution (and thereby nested-paging) for
9979	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
9980	* normally intercept #PFs, it might intercept them for debugging purposes.
9981	*
9982	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF
9983	* filters. If the outer guest is intercepting #PFs we must intercept all #PFs.
9984	*/
9985	uint32_t u32XcptPFMask;
9986	uint32_t u32XcptPFMatch;
9987	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
9988	{
9989	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
9990	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
9991	}
9992	else
9993	{
9994	u32XcptPFMask = 0;
9995	u32XcptPFMatch = 0;
9996	}
9997
9998	/*
9999	* Pause-Loop exiting.
10000	*/
10001	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10002	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10003
10004	/*
10005	* I/O Bitmap.
10006	*
10007	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we
10008	* always intercept all I/O port accesses.
10009	*/
10010	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10011
10012	/*
10013	* APIC-access page.
10014	*
10015	* The APIC-access page address has already been initialized while setting up the
10016	* nested-guest VMCS. In theory, even if the guest-physical address is invalid, it
10017	* should not be on any consequence to the host or to the guest for that matter, but
10018	* we only accept valid addresses verified by the VMLAUNCH/VMRESUME instruction
10019	* emulation to keep it simple.
10020	*/
10021
10022	/*
10023	* Virtual-APIC page and TPR threshold.
10024	*
10025	* We shall use the host-physical address of the virtual-APIC page in guest memory directly.
10026	* For this reason, we can access the virtual-APIC page of the nested-guest only using
10027	* PGM physical handlers as we must not assume a kernel virtual-address mapping exists and
10028	* requesting PGM for a mapping could be expensive/resource intensive (PGM mapping cache).
10029	*/
10030	RTHCPHYS HCPhysVirtApic = NIL_RTHCPHYS;
10031	uint32_t const u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10032	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10033	{
10034	int rc = PGMPhysGCPhys2HCPhys(pVM, pVmcsNstGst->u64AddrVirtApic.u, &HCPhysVirtApic);
10035
10036	/*
10037	* If the guest hypervisor has loaded crap into the virtual-APIC page field
10038	* we would fail to obtain a valid host-physical address for its guest-physical
10039	* address.
10040	*
10041	* We currently do not support this scenario. Maybe in the future if there is a
10042	* pressing need we can explore making this particular set of conditions work.
10043	* Right now we just cause a VM-entry failure.
10044	*
10045	* This has already been checked by VMLAUNCH/VMRESUME instruction emulation,
10046	* so should not really failure at the moment.
10047	*/
10048	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10049	}
10050	else
10051	{
10052	/*
10053	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10054	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10055	* be taken care of by EPT/shadow paging.
10056	*/
10057	if (pVM->hm.s.fAllow64BitGuests)
10058	{
10059	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10060	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10061	}
10062	}
10063
10064	/*
10065	* Validate basic assumptions.
10066	*/
10067	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10068	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10069	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10070	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10071
10072	/*
10073	* Commit it to the nested-guest VMCS.
10074	*/
10075	int rc = VINF_SUCCESS;
10076	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10077	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10078	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10079	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10080	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10081	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10082	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10083	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10084	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10085	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10086	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10087	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10088	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10089	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10090	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10091	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10092	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10093	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10094	{
10095	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10096	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10097	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10098	}
10099	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10100	{
10101	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10102	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10103	}
10104	AssertRCReturn(rc, rc);
10105
10106	/*
10107	* Update the nested-guest VMCS cache.
10108	*/
10109	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10110	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10111	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10112	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10113	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10114	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10115	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10116	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10117	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10118
10119	/*
10120	* MSR bitmap.
10121	*
10122	* The MSR bitmap address has already been initialized while setting up the
10123	* nested-guest VMCS, here we need to merge the MSR bitmaps.
10124	*/
10125	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10126	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10127
10128	return VINF_SUCCESS;
10129	}
10130	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10131
10132
10133	/**
10134	* Does the preparations before executing guest code in VT-x.
10135	*
10136	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10137	* recompiler/IEM. We must be cautious what we do here regarding committing
10138	* guest-state information into the VMCS assuming we assuredly execute the
10139	* guest in VT-x mode.
10140	*
10141	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10142	* the common-state (TRPM/forceflags), we must undo those changes so that the
10143	* recompiler/IEM can (and should) use them when it resumes guest execution.
10144	* Otherwise such operations must be done when we can no longer exit to ring-3.
10145	*
10146	* @returns Strict VBox status code (i.e. informational status codes too).
10147	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10148	* have been disabled.
10149	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10150	* double-fault into the guest.
10151	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10152	* dispatched directly.
10153	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10154	*
10155	* @param pVCpu The cross context virtual CPU structure.
10156	* @param pVmxTransient The VMX-transient structure.
10157	* @param fStepping Whether we are single-stepping the guest in the
10158	* hypervisor debugger. Makes us ignore some of the reasons
10159	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10160	* if event dispatching took place.
10161	*/
10162	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10163	{
10164	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10165
10166	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10167	if (pVmxTransient->fIsNestedGuest)
10168	{
10169	RT_NOREF2(pVCpu, fStepping);
10170	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10171	return VINF_EM_RESCHEDULE_REM;
10172	}
10173	#endif
10174
10175	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10176	PGMRZDynMapFlushAutoSet(pVCpu);
10177	#endif
10178
10179	/*
10180	* Check and process force flag actions, some of which might require us to go back to ring-3.
10181	*/
10182	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10183	if (rcStrict == VINF_SUCCESS)
10184	{ /* FFs don't get set all the time. */ }
10185	else
10186	return rcStrict;
10187
10188	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10189	/*
10190	* Switch to the nested-guest VMCS as we may have transitioned into executing
10191	* the nested-guest without leaving ring-0. Otherwise, if we came from ring-3
10192	* we would load the nested-guest VMCS while entering the VMX ring-0 session.
10193	*
10194	* We do this as late as possible to minimize (though not completely remove)
10195	* clearing/loading VMCS again due to premature trips to ring-3 above.
10196	*/
10197	if (pVmxTransient->fIsNestedGuest)
10198	{
10199	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10200	{
10201	/*
10202	* Ensure we have synced everything from the guest VMCS and also flag that
10203	* that we need to export the full (nested) guest-CPU context to the
10204	* nested-guest VMCS.
10205	*/
10206	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
10207	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST);
10208
10209	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
10210	int rc = hmR0VmxSwitchVmcs(&pVCpu->hm.s.vmx.VmcsInfo, &pVCpu->hm.s.vmx.VmcsInfoNstGst);
10211	if (RT_LIKELY(rc == VINF_SUCCESS))
10212	{
10213	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = true;
10214	ASMSetFlags(fEFlags);
10215	pVmxTransient->pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10216
10217	/*
10218	* We use a different VM-exit MSR-store area for the nested-guest. Hence,
10219	* flag that we need to update the host MSR values there. Even if we decide
10220	* in the future to share the VM-exit MSR-store area page with the guest,
10221	* if its content differs, we would have to update the host MSRs anyway.
10222	*/
10223	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
10224	Assert(!pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer); /** @todo NSTVMX: Paranoia remove later. */
10225	}
10226	else
10227	{
10228	ASMSetFlags(fEFlags);
10229	return rc;
10230	}
10231	}
10232
10233	/*
10234	* Merge guest VMCS controls with the nested-guest VMCS controls.
10235	*
10236	* Even if we have not executed the guest prior to this (e.g. when resuming
10237	* from a saved state), we should be okay with merging controls as we
10238	* initialize the guest VMCS controls as part of VM setup phase.
10239	*/
10240	if (!pVCpu->hm.s.vmx.fMergedNstGstCtls)
10241	{
10242	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10243	AssertRCReturn(rc, rc);
10244	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10245	}
10246	}
10247	#endif
10248
10249	/*
10250	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10251	* We look at the guest VMCS control here as we always set it when supported by
10252	* the physical CPU. Looking at the nested-guest control here would not be
10253	* possible because they are not merged yet.
10254	*/
10255	PVM pVM = pVCpu->CTX_SUFF(pVM);
10256	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10257	Assert(pVmcsInfo);
10258	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10259	&& (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10260	&& PDMHasApic(pVM))
10261	{
10262	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10263	AssertRCReturn(rc, rc);
10264	}
10265
10266	/*
10267	* Evaluate events to be injected into the guest.
10268	*
10269	* Events in TRPM can be injected without inspecting the guest state.
10270	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10271	* guest to cause a VM-exit the next time they are ready to receive the event.
10272	*/
10273	if (TRPMHasTrap(pVCpu))
10274	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10275
10276	uint32_t fIntrState;
10277	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10278
10279	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10280	/*
10281	* While evaluating pending events if something failed (unlikely) or if we were
10282	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10283	*/
10284	if ( rcStrict != VINF_SUCCESS
10285	\|\| ( pVmxTransient->fIsNestedGuest
10286	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)))
10287	return rcStrict;
10288	#endif
10289
10290	/*
10291	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10292	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10293	* also result in triple-faulting the VM.
10294	*
10295	* The above does not apply when executing a nested-guest (since unrestricted guest execution
10296	* is a requirement) regardless doing it avoid duplicating code elsewhere.
10297	*/
10298	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10299	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10300	{ /* likely */ }
10301	else
10302	{
10303	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10304	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10305	return rcStrict;
10306	}
10307
10308	/*
10309	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10310	* import CR3 themselves. We will need to update them here, as even as late as the above
10311	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10312	* the below force flags to be set.
10313	*/
10314	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10315	{
10316	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10317	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10318	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10319	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10320	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10321	}
10322	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10323	{
10324	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10325	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10326	}
10327
10328	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10329	/* Paranoia. */
10330	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10331	#endif
10332
10333	/*
10334	* No longjmps to ring-3 from this point on!!!
10335	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10336	* This also disables flushing of the R0-logger instance (if any).
10337	*/
10338	VMMRZCallRing3Disable(pVCpu);
10339
10340	/*
10341	* Export the guest state bits.
10342	*
10343	* We cannot perform longjmps while loading the guest state because we do not preserve the
10344	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10345	* CPU migration.
10346	*
10347	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10348	* registers. Hence, loading of the guest state needs to be done -after- injection of events.
10349	*/
10350	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10351	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10352	{ /* likely */ }
10353	else
10354	{
10355	VMMRZCallRing3Enable(pVCpu);
10356	return rcStrict;
10357	}
10358
10359	/*
10360	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10361	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10362	* preemption disabled for a while. Since this is purely to aid the
10363	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10364	* disable interrupt on NT.
10365	*
10366	* We need to check for force-flags that could've possible been altered since we last
10367	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10368	* see @bugref{6398}).
10369	*
10370	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10371	* to ring-3 before executing guest code.
10372	*/
10373	pVmxTransient->fEFlags = ASMIntDisableFlags();
10374
10375	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10376	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10377	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10378	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10379	{
10380	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10381	{
10382	pVCpu->hm.s.Event.fPending = false;
10383
10384	/*
10385	* We've injected any pending events. This is really the point of no return (to ring-3).
10386	*
10387	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10388	* returns from this function, so don't enable them here.
10389	*/
10390	return VINF_SUCCESS;
10391	}
10392
10393	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10394	rcStrict = VINF_EM_RAW_INTERRUPT;
10395	}
10396	else
10397	{
10398	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10399	rcStrict = VINF_EM_RAW_TO_R3;
10400	}
10401
10402	ASMSetFlags(pVmxTransient->fEFlags);
10403	VMMRZCallRing3Enable(pVCpu);
10404
10405	return rcStrict;
10406	}
10407
10408
10409	/**
10410	* Final preparations before executing guest code using hardware-assisted VMX.
10411	*
10412	* We can no longer get preempted to a different host CPU and there are no returns
10413	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10414	* failures), this function is not intended to fail sans unrecoverable hardware
10415	* errors.
10416	*
10417	* @param pVCpu The cross context virtual CPU structure.
10418	* @param pVmxTransient The VMX-transient structure.
10419	*
10420	* @remarks Called with preemption disabled.
10421	* @remarks No-long-jump zone!!!
10422	*/
10423	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10424	{
10425	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10426	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10427	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10428	Assert(!pVCpu->hm.s.Event.fPending);
10429
10430	/*
10431	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10432	*/
10433	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10434	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10435
10436	PVM pVM = pVCpu->CTX_SUFF(pVM);
10437	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10438
10439	if (!CPUMIsGuestFPUStateActive(pVCpu))
10440	{
10441	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10442	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10443	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10444	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10445	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10446	}
10447
10448	/*
10449	* Re-save the host state bits as we may've been preempted (only happens when
10450	* thread-context hooks are used or when the VM start function changes).
10451	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10452	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10453	* see @bugref{8432}.
10454	*
10455	* This may also happen when switching to/from a nested-guest VMCS without leaving
10456	* ring-0.
10457	*/
10458	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10459	{
10460	int rc = hmR0VmxExportHostState(pVCpu);
10461	AssertRC(rc);
10462	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
10463	}
10464	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10465
10466	/*
10467	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10468	*/
10469	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10470	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10471	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10472
10473	/*
10474	* Store status of the shared guest/host debug state at the time of VM-entry.
10475	*/
10476	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
10477	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
10478	{
10479	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
10480	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
10481	}
10482	else
10483	#endif
10484	{
10485	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10486	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10487	}
10488
10489	/*
10490	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10491	* more than one conditional check. The post-run side of our code shall determine
10492	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10493	*/
10494	if (pVmcsInfo->pbVirtApic)
10495	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10496
10497	/*
10498	* Update the host MSRs values in the VM-exit MSR-load area.
10499	*/
10500	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10501	{
10502	if (pVmcsInfo->cExitMsrLoad > 0)
10503	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10504	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10505	}
10506
10507	/*
10508	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10509	* VMX-preemption timer based on the next virtual sync clock deadline.
10510	*/
10511	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10512	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10513	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10514	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10515	{
10516	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10517	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10518	}
10519
10520	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10521	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10522	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10523	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
10524
10525	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10526
10527	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10528	as we're about to start executing the guest . */
10529
10530	/*
10531	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10532	*
10533	* This is done this late as updating the TSC offsetting/preemption timer above
10534	* figures out if we can skip intercepting RDTSCP by calculating the number of
10535	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10536	*/
10537	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10538	{
10539	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10540	{
10541	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10542	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10543	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10544	AssertRC(rc);
10545	}
10546	else
10547	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
10548	}
10549
10550	#ifdef VBOX_STRICT
10551	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo);
10552	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10553	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo));
10554	#endif
10555
10556	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10557	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10558	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs. */
10559	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10560	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10561	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10562	#endif
10563	}
10564
10565
10566	/**
10567	* First C routine invoked after running guest code using hardware-assisted VMX.
10568	*
10569	* @param pVCpu The cross context virtual CPU structure.
10570	* @param pVmxTransient The VMX-transient structure.
10571	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10572	*
10573	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10574	*
10575	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10576	* unconditionally when it is safe to do so.
10577	*/
10578	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10579	{
10580	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10581
10582	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10583	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10584	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10585	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10586	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10587	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10588
10589	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10590	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10591	{
10592	uint64_t uGstTsc;
10593	if (!pVmxTransient->fIsNestedGuest)
10594	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10595	else
10596	{
10597	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10598	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10599	}
10600	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10601	}
10602
10603	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10604	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
10605	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10606
10607	#if HC_ARCH_BITS == 64
10608	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10609	#endif
10610	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
10611	/* The 64-on-32 switcher maintains VMCS-launch state on its own
10612	and we need to leave it alone here. */
10613	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
10614	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10615	#else
10616	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10617	#endif
10618	#ifdef VBOX_STRICT
10619	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10620	#endif
10621	Assert(!ASMIntAreEnabled());
10622	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10623	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10624
10625	/*
10626	* Save the basic VM-exit reason and check if the VM-entry failed.
10627	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10628	*/
10629	uint32_t uExitReason;
10630	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
10631	AssertRC(rc);
10632	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
10633	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
10634
10635	/*
10636	* Check if VMLAUNCH/VMRESUME succeeded.
10637	* If this failed, we cause a guru meditation and cease further execution.
10638	*/
10639	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
10640	{
10641	/*
10642	* Update the VM-exit history array here even if the VM-entry failed due to:
10643	* - Invalid guest state.
10644	* - MSR loading.
10645	* - Machine-check event.
10646	*
10647	* In any of the above cases we will still have a "valid" VM-exit reason
10648	* despite @a fVMEntryFailed being false.
10649	*
10650	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
10651	*
10652	* Note! We don't have CS or RIP at this point. Will probably address that later
10653	* by amending the history entry added here.
10654	*/
10655	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
10656	UINT64_MAX, uHostTsc);
10657
10658	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
10659	{
10660	VMMRZCallRing3Enable(pVCpu);
10661
10662	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10663	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10664
10665	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
10666	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10667	AssertRC(rc);
10668	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
10669	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
10670	AssertRC(rc);
10671	#else
10672	/*
10673	* Import the guest-interruptibility state always as we need it while evaluating
10674	* injecting events on re-entry.
10675	*
10676	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
10677	* checking for real-mode while exporting the state because all bits that cause
10678	* mode changes wrt CR0 are intercepted.
10679	*/
10680	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
10681	AssertRC(rc);
10682	#endif
10683
10684	/*
10685	* Sync the TPR shadow with our APIC state.
10686	*/
10687	if ( !pVmxTransient->fIsNestedGuest
10688	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
10689	{
10690	Assert(pVmcsInfo->pbVirtApic);
10691	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
10692	{
10693	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
10694	AssertRC(rc);
10695	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
10696	}
10697	}
10698
10699	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10700	return;
10701	}
10702	}
10703	else
10704	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
10705
10706	VMMRZCallRing3Enable(pVCpu);
10707	}
10708
10709
10710	/**
10711	* Runs the guest code using hardware-assisted VMX the normal way.
10712	*
10713	* @returns VBox status code.
10714	* @param pVCpu The cross context virtual CPU structure.
10715	* @param pcLoops Pointer to the number of executed loops.
10716	*/
10717	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
10718	{
10719	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10720	Assert(pcLoops);
10721	Assert(*pcLoops <= cMaxResumeLoops);
10722
10723	VMXTRANSIENT VmxTransient;
10724	RT_ZERO(VmxTransient);
10725	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10726
10727	/* Paranoia. */
10728	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
10729	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10730
10731	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10732	for (;;)
10733	{
10734	Assert(!HMR0SuspendPending());
10735	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10736	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10737
10738	/*
10739	* Preparatory work for running nested-guest code, this may force us to
10740	* return to ring-3.
10741	*
10742	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10743	*/
10744	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10745	if (rcStrict != VINF_SUCCESS)
10746	break;
10747
10748	/* Interrupts are disabled at this point! */
10749	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10750	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10751	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10752	/* Interrupts are re-enabled at this point! */
10753
10754	/*
10755	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10756	*/
10757	if (RT_SUCCESS(rcRun))
10758	{ /* very likely */ }
10759	else
10760	{
10761	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10762	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10763	return rcRun;
10764	}
10765
10766	/*
10767	* Profile the VM-exit.
10768	*/
10769	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10770	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10771	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10772	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10773	HMVMX_START_EXIT_DISPATCH_PROF();
10774
10775	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10776
10777	/*
10778	* Handle the VM-exit.
10779	*/
10780	#ifdef HMVMX_USE_FUNCTION_TABLE
10781	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
10782	#else
10783	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
10784	#endif
10785	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10786	if (rcStrict == VINF_SUCCESS)
10787	{
10788	if (++(*pcLoops) <= cMaxResumeLoops)
10789	continue;
10790	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10791	rcStrict = VINF_EM_RAW_INTERRUPT;
10792	}
10793	break;
10794	}
10795
10796	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10797	return rcStrict;
10798	}
10799
10800	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10801	/**
10802	* Runs the nested-guest code using hardware-assisted VMX.
10803	*
10804	* @returns VBox status code.
10805	* @param pVCpu The cross context virtual CPU structure.
10806	* @param pcLoops Pointer to the number of executed loops.
10807	*
10808	* @sa hmR0VmxRunGuestCodeNormal().
10809	*/
10810	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
10811	{
10812	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10813	Assert(pcLoops);
10814	Assert(*pcLoops <= cMaxResumeLoops);
10815
10816	VMXTRANSIENT VmxTransient;
10817	RT_ZERO(VmxTransient);
10818	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10819	VmxTransient.fIsNestedGuest = true;
10820
10821	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10822	for (;;)
10823	{
10824	Assert(!HMR0SuspendPending());
10825	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10826	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10827
10828	/*
10829	* Preparatory work for running guest code, this may force us to
10830	* return to ring-3.
10831	*
10832	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10833	*/
10834	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10835	if (rcStrict != VINF_SUCCESS)
10836	break;
10837
10838	/* Interrupts are disabled at this point! */
10839	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10840	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10841	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10842	/* Interrupts are re-enabled at this point! */
10843
10844	/*
10845	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10846	*/
10847	if (RT_SUCCESS(rcRun))
10848	{ /* very likely */ }
10849	else
10850	{
10851	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10852	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10853	return rcRun;
10854	}
10855
10856	/*
10857	* Profile the VM-exit.
10858	*/
10859	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10860	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10861	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10862	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10863	HMVMX_START_EXIT_DISPATCH_PROF();
10864
10865	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10866
10867	/*
10868	* Handle the VM-exit.
10869	*/
10870	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
10871	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10872	if ( rcStrict == VINF_SUCCESS
10873	&& CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10874	{
10875	if (++(*pcLoops) <= cMaxResumeLoops)
10876	continue;
10877	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10878	rcStrict = VINF_EM_RAW_INTERRUPT;
10879	}
10880	break;
10881	}
10882
10883	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10884	return rcStrict;
10885	}
10886	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10887
10888
10889	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
10890	* probes.
10891	*
10892	* The following few functions and associated structure contains the bloat
10893	* necessary for providing detailed debug events and dtrace probes as well as
10894	* reliable host side single stepping. This works on the principle of
10895	* "subclassing" the normal execution loop and workers. We replace the loop
10896	* method completely and override selected helpers to add necessary adjustments
10897	* to their core operation.
10898	*
10899	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
10900	* any performance for debug and analysis features.
10901	*
10902	* @{
10903	*/
10904
10905	/**
10906	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
10907	* the debug run loop.
10908	*/
10909	typedef struct VMXRUNDBGSTATE
10910	{
10911	/** The RIP we started executing at. This is for detecting that we stepped. */
10912	uint64_t uRipStart;
10913	/** The CS we started executing with. */
10914	uint16_t uCsStart;
10915
10916	/** Whether we've actually modified the 1st execution control field. */
10917	bool fModifiedProcCtls : 1;
10918	/** Whether we've actually modified the 2nd execution control field. */
10919	bool fModifiedProcCtls2 : 1;
10920	/** Whether we've actually modified the exception bitmap. */
10921	bool fModifiedXcptBitmap : 1;
10922
10923	/** We desire the modified the CR0 mask to be cleared. */
10924	bool fClearCr0Mask : 1;
10925	/** We desire the modified the CR4 mask to be cleared. */
10926	bool fClearCr4Mask : 1;
10927	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
10928	uint32_t fCpe1Extra;
10929	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
10930	uint32_t fCpe1Unwanted;
10931	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
10932	uint32_t fCpe2Extra;
10933	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
10934	uint32_t bmXcptExtra;
10935	/** The sequence number of the Dtrace provider settings the state was
10936	* configured against. */
10937	uint32_t uDtraceSettingsSeqNo;
10938	/** VM-exits to check (one bit per VM-exit). */
10939	uint32_t bmExitsToCheck[3];
10940
10941	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
10942	uint32_t fProcCtlsInitial;
10943	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
10944	uint32_t fProcCtls2Initial;
10945	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
10946	uint32_t bmXcptInitial;
10947	} VMXRUNDBGSTATE;
10948	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
10949	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
10950
10951
10952	/**
10953	* Initializes the VMXRUNDBGSTATE structure.
10954	*
10955	* @param pVCpu The cross context virtual CPU structure of the
10956	* calling EMT.
10957	* @param pVmxTransient The VMX-transient structure.
10958	* @param pDbgState The debug state to initialize.
10959	*/
10960	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
10961	{
10962	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
10963	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
10964
10965	pDbgState->fModifiedProcCtls = false;
10966	pDbgState->fModifiedProcCtls2 = false;
10967	pDbgState->fModifiedXcptBitmap = false;
10968	pDbgState->fClearCr0Mask = false;
10969	pDbgState->fClearCr4Mask = false;
10970	pDbgState->fCpe1Extra = 0;
10971	pDbgState->fCpe1Unwanted = 0;
10972	pDbgState->fCpe2Extra = 0;
10973	pDbgState->bmXcptExtra = 0;
10974	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
10975	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
10976	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
10977	}
10978
10979
10980	/**
10981	* Updates the VMSC fields with changes requested by @a pDbgState.
10982	*
10983	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
10984	* immediately before executing guest code, i.e. when interrupts are disabled.
10985	* We don't check status codes here as we cannot easily assert or return in the
10986	* latter case.
10987	*
10988	* @param pVCpu The cross context virtual CPU structure.
10989	* @param pVmxTransient The VMX-transient structure.
10990	* @param pDbgState The debug state.
10991	*/
10992	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
10993	{
10994	/*
10995	* Ensure desired flags in VMCS control fields are set.
10996	* (Ignoring write failure here, as we're committed and it's just debug extras.)
10997	*
10998	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
10999	* there should be no stale data in pCtx at this point.
11000	*/
11001	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11002	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11003	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11004	{
11005	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11006	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11007	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11008	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11009	pDbgState->fModifiedProcCtls = true;
11010	}
11011
11012	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11013	{
11014	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11015	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11016	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11017	pDbgState->fModifiedProcCtls2 = true;
11018	}
11019
11020	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11021	{
11022	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11023	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11024	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11025	pDbgState->fModifiedXcptBitmap = true;
11026	}
11027
11028	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11029	{
11030	pVmcsInfo->u64Cr0Mask = 0;
11031	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
11032	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11033	}
11034
11035	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11036	{
11037	pVmcsInfo->u64Cr4Mask = 0;
11038	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
11039	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11040	}
11041
11042	NOREF(pVCpu);
11043	}
11044
11045
11046	/**
11047	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11048	* re-entry next time around.
11049	*
11050	* @returns Strict VBox status code (i.e. informational status codes too).
11051	* @param pVCpu The cross context virtual CPU structure.
11052	* @param pVmxTransient The VMX-transient structure.
11053	* @param pDbgState The debug state.
11054	* @param rcStrict The return code from executing the guest using single
11055	* stepping.
11056	*/
11057	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11058	VBOXSTRICTRC rcStrict)
11059	{
11060	/*
11061	* Restore VM-exit control settings as we may not reenter this function the
11062	* next time around.
11063	*/
11064	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11065
11066	/* We reload the initial value, trigger what we can of recalculations the
11067	next time around. From the looks of things, that's all that's required atm. */
11068	if (pDbgState->fModifiedProcCtls)
11069	{
11070	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11071	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11072	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11073	AssertRCReturn(rc2, rc2);
11074	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11075	}
11076
11077	/* We're currently the only ones messing with this one, so just restore the
11078	cached value and reload the field. */
11079	if ( pDbgState->fModifiedProcCtls2
11080	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11081	{
11082	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11083	AssertRCReturn(rc2, rc2);
11084	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11085	}
11086
11087	/* If we've modified the exception bitmap, we restore it and trigger
11088	reloading and partial recalculation the next time around. */
11089	if (pDbgState->fModifiedXcptBitmap)
11090	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11091
11092	return rcStrict;
11093	}
11094
11095
11096	/**
11097	* Configures VM-exit controls for current DBGF and DTrace settings.
11098	*
11099	* This updates @a pDbgState and the VMCS execution control fields to reflect
11100	* the necessary VM-exits demanded by DBGF and DTrace.
11101	*
11102	* @param pVCpu The cross context virtual CPU structure.
11103	* @param pVmxTransient The VMX-transient structure. May update
11104	* fUpdatedTscOffsettingAndPreemptTimer.
11105	* @param pDbgState The debug state.
11106	*/
11107	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11108	{
11109	/*
11110	* Take down the dtrace serial number so we can spot changes.
11111	*/
11112	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11113	ASMCompilerBarrier();
11114
11115	/*
11116	* We'll rebuild most of the middle block of data members (holding the
11117	* current settings) as we go along here, so start by clearing it all.
11118	*/
11119	pDbgState->bmXcptExtra = 0;
11120	pDbgState->fCpe1Extra = 0;
11121	pDbgState->fCpe1Unwanted = 0;
11122	pDbgState->fCpe2Extra = 0;
11123	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11124	pDbgState->bmExitsToCheck[i] = 0;
11125
11126	/*
11127	* Software interrupts (INT XXh) - no idea how to trigger these...
11128	*/
11129	PVM pVM = pVCpu->CTX_SUFF(pVM);
11130	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11131	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11132	{
11133	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11134	}
11135
11136	/*
11137	* INT3 breakpoints - triggered by #BP exceptions.
11138	*/
11139	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11140	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11141
11142	/*
11143	* Exception bitmap and XCPT events+probes.
11144	*/
11145	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11146	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11147	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11148
11149	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11150	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11151	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11152	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11153	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11154	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11155	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11156	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11157	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11158	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11159	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11160	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11161	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11162	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11163	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11164	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11165	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11166	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11167
11168	if (pDbgState->bmXcptExtra)
11169	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11170
11171	/*
11172	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11173	*
11174	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11175	* So, when adding/changing/removing please don't forget to update it.
11176	*
11177	* Some of the macros are picking up local variables to save horizontal space,
11178	* (being able to see it in a table is the lesser evil here).
11179	*/
11180	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11181	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11182	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11183	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11184	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11185	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11186	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11187	} else do { } while (0)
11188	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11189	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11190	{ \
11191	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11192	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11193	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11194	} else do { } while (0)
11195	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11196	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11197	{ \
11198	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11199	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11200	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11201	} else do { } while (0)
11202	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11203	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11204	{ \
11205	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11206	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11207	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11208	} else do { } while (0)
11209
11210	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11211	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11212	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11213	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11214	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11215
11216	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11217	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11218	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11219	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11220	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11221	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11222	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11223	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11224	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11225	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11226	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11227	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11228	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11229	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11230	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11231	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11232	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11233	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11234	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11235	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11236	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11237	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11238	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11239	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11240	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11241	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11242	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11243	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11244	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11245	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11246	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11247	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11248	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11249	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11250	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11251	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11252
11253	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11254	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11255	{
11256	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11257	\| CPUMCTX_EXTRN_APIC_TPR);
11258	AssertRC(rc);
11259
11260	#if 0 /** @todo fix me */
11261	pDbgState->fClearCr0Mask = true;
11262	pDbgState->fClearCr4Mask = true;
11263	#endif
11264	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11265	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11266	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11267	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11268	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11269	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11270	require clearing here and in the loop if we start using it. */
11271	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11272	}
11273	else
11274	{
11275	if (pDbgState->fClearCr0Mask)
11276	{
11277	pDbgState->fClearCr0Mask = false;
11278	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11279	}
11280	if (pDbgState->fClearCr4Mask)
11281	{
11282	pDbgState->fClearCr4Mask = false;
11283	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11284	}
11285	}
11286	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11287	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11288
11289	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11290	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11291	{
11292	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11293	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11294	}
11295	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11296	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11297
11298	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11299	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11300	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11301	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11302	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11303	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11304	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11305	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11306	#if 0 /** @todo too slow, fix handler. */
11307	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11308	#endif
11309	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11310
11311	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11312	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11313	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11314	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11315	{
11316	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11317	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11318	}
11319	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11320	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11321	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11322	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11323
11324	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11325	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11326	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11327	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11328	{
11329	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11330	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11331	}
11332	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11333	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11334	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11335	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11336
11337	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11338	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11339	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11340	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11341	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11342	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11343	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11344	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11345	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11346	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11347	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11348	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11349	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11350	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11351	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11352	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11353	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11354	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11355	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11356	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11357	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11358	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11359
11360	#undef IS_EITHER_ENABLED
11361	#undef SET_ONLY_XBM_IF_EITHER_EN
11362	#undef SET_CPE1_XBM_IF_EITHER_EN
11363	#undef SET_CPEU_XBM_IF_EITHER_EN
11364	#undef SET_CPE2_XBM_IF_EITHER_EN
11365
11366	/*
11367	* Sanitize the control stuff.
11368	*/
11369	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11370	if (pDbgState->fCpe2Extra)
11371	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11372	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11373	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11374	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11375	{
11376	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11377	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11378	}
11379
11380	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11381	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11382	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11383	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11384	}
11385
11386
11387	/**
11388	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11389	* appropriate.
11390	*
11391	* The caller has checked the VM-exit against the
11392	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11393	* already, so we don't have to do that either.
11394	*
11395	* @returns Strict VBox status code (i.e. informational status codes too).
11396	* @param pVCpu The cross context virtual CPU structure.
11397	* @param pVmxTransient The VMX-transient structure.
11398	* @param uExitReason The VM-exit reason.
11399	*
11400	* @remarks The name of this function is displayed by dtrace, so keep it short
11401	* and to the point. No longer than 33 chars long, please.
11402	*/
11403	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11404	{
11405	/*
11406	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11407	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11408	*
11409	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11410	* does. Must add/change/remove both places. Same ordering, please.
11411	*
11412	* Added/removed events must also be reflected in the next section
11413	* where we dispatch dtrace events.
11414	*/
11415	bool fDtrace1 = false;
11416	bool fDtrace2 = false;
11417	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11418	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11419	uint32_t uEventArg = 0;
11420	#define SET_EXIT(a_EventSubName) \
11421	do { \
11422	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11423	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11424	} while (0)
11425	#define SET_BOTH(a_EventSubName) \
11426	do { \
11427	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11428	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11429	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11430	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11431	} while (0)
11432	switch (uExitReason)
11433	{
11434	case VMX_EXIT_MTF:
11435	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11436
11437	case VMX_EXIT_XCPT_OR_NMI:
11438	{
11439	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11440	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11441	{
11442	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11443	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11444	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11445	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11446	{
11447	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11448	{
11449	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11450	uEventArg = pVmxTransient->uExitIntErrorCode;
11451	}
11452	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11453	switch (enmEvent1)
11454	{
11455	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11456	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11457	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11458	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11459	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11460	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11461	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11462	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11463	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11464	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11465	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11466	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11467	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11468	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11469	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11470	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11471	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11472	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11473	default: break;
11474	}
11475	}
11476	else
11477	AssertFailed();
11478	break;
11479
11480	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11481	uEventArg = idxVector;
11482	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11483	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11484	break;
11485	}
11486	break;
11487	}
11488
11489	case VMX_EXIT_TRIPLE_FAULT:
11490	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11491	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11492	break;
11493	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11494	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11495	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11496	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11497	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11498
11499	/* Instruction specific VM-exits: */
11500	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11501	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11502	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11503	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11504	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11505	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11506	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11507	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11508	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11509	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11510	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11511	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11512	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11513	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11514	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11515	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11516	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11517	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11518	case VMX_EXIT_MOV_CRX:
11519	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11520	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11521	SET_BOTH(CRX_READ);
11522	else
11523	SET_BOTH(CRX_WRITE);
11524	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11525	break;
11526	case VMX_EXIT_MOV_DRX:
11527	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11528	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11529	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11530	SET_BOTH(DRX_READ);
11531	else
11532	SET_BOTH(DRX_WRITE);
11533	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11534	break;
11535	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11536	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11537	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11538	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11539	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11540	case VMX_EXIT_GDTR_IDTR_ACCESS:
11541	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11542	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11543	{
11544	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11545	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11546	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11547	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11548	}
11549	break;
11550
11551	case VMX_EXIT_LDTR_TR_ACCESS:
11552	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11553	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11554	{
11555	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11556	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11557	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11558	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11559	}
11560	break;
11561
11562	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11563	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11564	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11565	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11566	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11567	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11568	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11569	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11570	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11571	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11572	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11573
11574	/* Events that aren't relevant at this point. */
11575	case VMX_EXIT_EXT_INT:
11576	case VMX_EXIT_INT_WINDOW:
11577	case VMX_EXIT_NMI_WINDOW:
11578	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11579	case VMX_EXIT_PREEMPT_TIMER:
11580	case VMX_EXIT_IO_INSTR:
11581	break;
11582
11583	/* Errors and unexpected events. */
11584	case VMX_EXIT_INIT_SIGNAL:
11585	case VMX_EXIT_SIPI:
11586	case VMX_EXIT_IO_SMI:
11587	case VMX_EXIT_SMI:
11588	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11589	case VMX_EXIT_ERR_MSR_LOAD:
11590	case VMX_EXIT_ERR_MACHINE_CHECK:
11591	break;
11592
11593	default:
11594	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11595	break;
11596	}
11597	#undef SET_BOTH
11598	#undef SET_EXIT
11599
11600	/*
11601	* Dtrace tracepoints go first. We do them here at once so we don't
11602	* have to copy the guest state saving and stuff a few dozen times.
11603	* Down side is that we've got to repeat the switch, though this time
11604	* we use enmEvent since the probes are a subset of what DBGF does.
11605	*/
11606	if (fDtrace1 \|\| fDtrace2)
11607	{
11608	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11609	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11610	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11611	switch (enmEvent1)
11612	{
11613	/** @todo consider which extra parameters would be helpful for each probe. */
11614	case DBGFEVENT_END: break;
11615	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11616	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11617	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11618	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11619	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11620	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11621	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11622	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11623	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
11624	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
11625	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
11626	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
11627	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
11628	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
11629	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
11630	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
11631	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
11632	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
11633	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11634	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11635	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
11636	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
11637	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
11638	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
11639	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
11640	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
11641	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
11642	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11643	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11644	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11645	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11646	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11647	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
11648	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11649	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
11650	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
11651	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
11652	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
11653	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
11654	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
11655	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
11656	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
11657	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
11658	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
11659	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
11660	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
11661	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
11662	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
11663	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
11664	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
11665	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
11666	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
11667	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
11668	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
11669	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
11670	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
11671	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
11672	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
11673	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
11674	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
11675	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
11676	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
11677	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
11678	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
11679	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
11680	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
11681	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
11682	}
11683	switch (enmEvent2)
11684	{
11685	/** @todo consider which extra parameters would be helpful for each probe. */
11686	case DBGFEVENT_END: break;
11687	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
11688	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11689	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
11690	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
11691	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
11692	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
11693	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
11694	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
11695	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
11696	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11697	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11698	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11699	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11700	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11701	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
11702	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11703	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
11704	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
11705	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
11706	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
11707	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
11708	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
11709	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
11710	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
11711	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
11712	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
11713	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
11714	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
11715	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
11716	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
11717	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
11718	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
11719	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
11720	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
11721	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
11722	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
11723	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
11724	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
11725	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
11726	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
11727	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
11728	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
11729	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
11730	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
11731	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
11732	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
11733	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
11734	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
11735	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
11736	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
11737	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
11738	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
11739	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
11740	}
11741	}
11742
11743	/*
11744	* Fire of the DBGF event, if enabled (our check here is just a quick one,
11745	* the DBGF call will do a full check).
11746	*
11747	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
11748	* Note! If we have to events, we prioritize the first, i.e. the instruction
11749	* one, in order to avoid event nesting.
11750	*/
11751	PVM pVM = pVCpu->CTX_SUFF(pVM);
11752	if ( enmEvent1 != DBGFEVENT_END
11753	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
11754	{
11755	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11756	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
11757	if (rcStrict != VINF_SUCCESS)
11758	return rcStrict;
11759	}
11760	else if ( enmEvent2 != DBGFEVENT_END
11761	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
11762	{
11763	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11764	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
11765	if (rcStrict != VINF_SUCCESS)
11766	return rcStrict;
11767	}
11768
11769	return VINF_SUCCESS;
11770	}
11771
11772
11773	/**
11774	* Single-stepping VM-exit filtering.
11775	*
11776	* This is preprocessing the VM-exits and deciding whether we've gotten far
11777	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
11778	* handling is performed.
11779	*
11780	* @returns Strict VBox status code (i.e. informational status codes too).
11781	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
11782	* @param pVmxTransient The VMX-transient structure.
11783	* @param pDbgState The debug state.
11784	*/
11785	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11786	{
11787	/*
11788	* Expensive (saves context) generic dtrace VM-exit probe.
11789	*/
11790	uint32_t const uExitReason = pVmxTransient->uExitReason;
11791	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
11792	{ /* more likely */ }
11793	else
11794	{
11795	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11796	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11797	AssertRC(rc);
11798	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
11799	}
11800
11801	/*
11802	* Check for host NMI, just to get that out of the way.
11803	*/
11804	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
11805	{ /* normally likely */ }
11806	else
11807	{
11808	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11809	AssertRCReturn(rc2, rc2);
11810	uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11811	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11812	return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
11813	}
11814
11815	/*
11816	* Check for single stepping event if we're stepping.
11817	*/
11818	if (pVCpu->hm.s.fSingleInstruction)
11819	{
11820	switch (uExitReason)
11821	{
11822	case VMX_EXIT_MTF:
11823	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11824
11825	/* Various events: */
11826	case VMX_EXIT_XCPT_OR_NMI:
11827	case VMX_EXIT_EXT_INT:
11828	case VMX_EXIT_TRIPLE_FAULT:
11829	case VMX_EXIT_INT_WINDOW:
11830	case VMX_EXIT_NMI_WINDOW:
11831	case VMX_EXIT_TASK_SWITCH:
11832	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11833	case VMX_EXIT_APIC_ACCESS:
11834	case VMX_EXIT_EPT_VIOLATION:
11835	case VMX_EXIT_EPT_MISCONFIG:
11836	case VMX_EXIT_PREEMPT_TIMER:
11837
11838	/* Instruction specific VM-exits: */
11839	case VMX_EXIT_CPUID:
11840	case VMX_EXIT_GETSEC:
11841	case VMX_EXIT_HLT:
11842	case VMX_EXIT_INVD:
11843	case VMX_EXIT_INVLPG:
11844	case VMX_EXIT_RDPMC:
11845	case VMX_EXIT_RDTSC:
11846	case VMX_EXIT_RSM:
11847	case VMX_EXIT_VMCALL:
11848	case VMX_EXIT_VMCLEAR:
11849	case VMX_EXIT_VMLAUNCH:
11850	case VMX_EXIT_VMPTRLD:
11851	case VMX_EXIT_VMPTRST:
11852	case VMX_EXIT_VMREAD:
11853	case VMX_EXIT_VMRESUME:
11854	case VMX_EXIT_VMWRITE:
11855	case VMX_EXIT_VMXOFF:
11856	case VMX_EXIT_VMXON:
11857	case VMX_EXIT_MOV_CRX:
11858	case VMX_EXIT_MOV_DRX:
11859	case VMX_EXIT_IO_INSTR:
11860	case VMX_EXIT_RDMSR:
11861	case VMX_EXIT_WRMSR:
11862	case VMX_EXIT_MWAIT:
11863	case VMX_EXIT_MONITOR:
11864	case VMX_EXIT_PAUSE:
11865	case VMX_EXIT_GDTR_IDTR_ACCESS:
11866	case VMX_EXIT_LDTR_TR_ACCESS:
11867	case VMX_EXIT_INVEPT:
11868	case VMX_EXIT_RDTSCP:
11869	case VMX_EXIT_INVVPID:
11870	case VMX_EXIT_WBINVD:
11871	case VMX_EXIT_XSETBV:
11872	case VMX_EXIT_RDRAND:
11873	case VMX_EXIT_INVPCID:
11874	case VMX_EXIT_VMFUNC:
11875	case VMX_EXIT_RDSEED:
11876	case VMX_EXIT_XSAVES:
11877	case VMX_EXIT_XRSTORS:
11878	{
11879	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11880	AssertRCReturn(rc, rc);
11881	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
11882	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
11883	return VINF_EM_DBG_STEPPED;
11884	break;
11885	}
11886
11887	/* Errors and unexpected events: */
11888	case VMX_EXIT_INIT_SIGNAL:
11889	case VMX_EXIT_SIPI:
11890	case VMX_EXIT_IO_SMI:
11891	case VMX_EXIT_SMI:
11892	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11893	case VMX_EXIT_ERR_MSR_LOAD:
11894	case VMX_EXIT_ERR_MACHINE_CHECK:
11895	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
11896	break;
11897
11898	default:
11899	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11900	break;
11901	}
11902	}
11903
11904	/*
11905	* Check for debugger event breakpoints and dtrace probes.
11906	*/
11907	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
11908	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
11909	{
11910	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
11911	if (rcStrict != VINF_SUCCESS)
11912	return rcStrict;
11913	}
11914
11915	/*
11916	* Normal processing.
11917	*/
11918	#ifdef HMVMX_USE_FUNCTION_TABLE
11919	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
11920	#else
11921	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
11922	#endif
11923	}
11924
11925
11926	/**
11927	* Single steps guest code using hardware-assisted VMX.
11928	*
11929	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
11930	* but single-stepping through the hypervisor debugger.
11931	*
11932	* @returns Strict VBox status code (i.e. informational status codes too).
11933	* @param pVCpu The cross context virtual CPU structure.
11934	* @param pcLoops Pointer to the number of executed loops.
11935	*
11936	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
11937	*/
11938	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
11939	{
11940	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11941	Assert(pcLoops);
11942	Assert(*pcLoops <= cMaxResumeLoops);
11943
11944	VMXTRANSIENT VmxTransient;
11945	RT_ZERO(VmxTransient);
11946	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11947
11948	/* Set HMCPU indicators. */
11949	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
11950	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
11951	pVCpu->hm.s.fDebugWantRdTscExit = false;
11952	pVCpu->hm.s.fUsingDebugLoop = true;
11953
11954	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
11955	VMXRUNDBGSTATE DbgState;
11956	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
11957	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
11958
11959	/*
11960	* The loop.
11961	*/
11962	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11963	for (;;)
11964	{
11965	Assert(!HMR0SuspendPending());
11966	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11967	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11968	bool fStepping = pVCpu->hm.s.fSingleInstruction;
11969
11970	/* Set up VM-execution controls the next two can respond to. */
11971	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
11972
11973	/*
11974	* Preparatory work for running guest code, this may force us to
11975	* return to ring-3.
11976	*
11977	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11978	*/
11979	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
11980	if (rcStrict != VINF_SUCCESS)
11981	break;
11982
11983	/* Interrupts are disabled at this point! */
11984	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11985
11986	/* Override any obnoxious code in the above two calls. */
11987	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
11988
11989	/*
11990	* Finally execute the guest.
11991	*/
11992	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11993
11994	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11995	/* Interrupts are re-enabled at this point! */
11996
11997	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
11998	if (RT_SUCCESS(rcRun))
11999	{ /* very likely */ }
12000	else
12001	{
12002	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12003	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12004	return rcRun;
12005	}
12006
12007	/* Profile the VM-exit. */
12008	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12009	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12010	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12011	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12012	HMVMX_START_EXIT_DISPATCH_PROF();
12013
12014	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12015
12016	/*
12017	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12018	*/
12019	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12020	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12021	if (rcStrict != VINF_SUCCESS)
12022	break;
12023	if (++(*pcLoops) > cMaxResumeLoops)
12024	{
12025	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12026	rcStrict = VINF_EM_RAW_INTERRUPT;
12027	break;
12028	}
12029
12030	/*
12031	* Stepping: Did the RIP change, if so, consider it a single step.
12032	* Otherwise, make sure one of the TFs gets set.
12033	*/
12034	if (fStepping)
12035	{
12036	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12037	AssertRC(rc);
12038	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12039	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12040	{
12041	rcStrict = VINF_EM_DBG_STEPPED;
12042	break;
12043	}
12044	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12045	}
12046
12047	/*
12048	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12049	*/
12050	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12051	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12052	}
12053
12054	/*
12055	* Clear the X86_EFL_TF if necessary.
12056	*/
12057	if (pVCpu->hm.s.fClearTrapFlag)
12058	{
12059	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12060	AssertRC(rc);
12061	pVCpu->hm.s.fClearTrapFlag = false;
12062	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12063	}
12064	/** @todo there seems to be issues with the resume flag when the monitor trap
12065	* flag is pending without being used. Seen early in bios init when
12066	* accessing APIC page in protected mode. */
12067
12068	/*
12069	* Restore VM-exit control settings as we may not re-enter this function the
12070	* next time around.
12071	*/
12072	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12073
12074	/* Restore HMCPU indicators. */
12075	pVCpu->hm.s.fUsingDebugLoop = false;
12076	pVCpu->hm.s.fDebugWantRdTscExit = false;
12077	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12078
12079	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12080	return rcStrict;
12081	}
12082
12083
12084	/** @} */
12085
12086
12087	/**
12088	* Checks if any expensive dtrace probes are enabled and we should go to the
12089	* debug loop.
12090	*
12091	* @returns true if we should use debug loop, false if not.
12092	*/
12093	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12094	{
12095	/* It's probably faster to OR the raw 32-bit counter variables together.
12096	Since the variables are in an array and the probes are next to one
12097	another (more or less), we have good locality. So, better read
12098	eight-nine cache lines ever time and only have one conditional, than
12099	128+ conditionals, right? */
12100	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12101	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12102	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12103	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12104	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12105	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12106	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12107	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12108	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12109	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12110	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12111	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12112	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12113	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12114	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12115	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12116	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12117	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12118	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12119	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12120	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12121	) != 0
12122	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12123	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12124	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12125	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12126	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12127	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12128	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12129	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12130	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12131	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12132	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12133	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12134	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12135	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12136	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12137	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12138	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12139	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12140	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12141	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12142	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12143	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12144	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12145	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12146	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12147	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12148	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12149	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12150	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12151	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12152	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12153	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12154	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12155	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12156	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12157	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12158	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12159	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12160	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12161	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12162	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12163	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12164	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12165	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12166	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12167	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12168	) != 0
12169	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12170	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12171	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12172	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12173	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12174	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12175	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12176	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12177	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12178	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12179	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12180	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12181	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12182	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12183	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12184	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12185	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12186	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12187	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12188	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12189	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12190	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12191	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12192	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12193	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12194	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12195	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12196	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12197	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12198	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12199	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12200	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12201	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12202	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12203	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12204	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12205	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12206	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12207	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12208	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12209	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12210	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12211	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12212	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12213	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12214	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12215	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12216	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12217	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12218	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12219	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12220	) != 0;
12221	}
12222
12223
12224	/**
12225	* Runs the guest using hardware-assisted VMX.
12226	*
12227	* @returns Strict VBox status code (i.e. informational status codes too).
12228	* @param pVCpu The cross context virtual CPU structure.
12229	*/
12230	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
12231	{
12232	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12233	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12234	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12235	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12236
12237	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
12238
12239	VBOXSTRICTRC rcStrict;
12240	uint32_t cLoops = 0;
12241	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12242	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12243	#else
12244	bool const fInNestedGuestMode = false;
12245	#endif
12246	if (!fInNestedGuestMode)
12247	{
12248	if ( !pVCpu->hm.s.fUseDebugLoop
12249	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12250	&& !DBGFIsStepping(pVCpu)
12251	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12252	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12253	else
12254	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12255	}
12256	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12257	else
12258	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
12259
12260	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12261	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12262	#endif
12263
12264	if (rcStrict == VERR_EM_INTERPRETER)
12265	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12266	else if (rcStrict == VINF_EM_RESET)
12267	rcStrict = VINF_EM_TRIPLE_FAULT;
12268
12269	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12270	if (RT_FAILURE(rc2))
12271	{
12272	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12273	rcStrict = rc2;
12274	}
12275	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12276	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12277	return rcStrict;
12278	}
12279
12280
12281	#ifndef HMVMX_USE_FUNCTION_TABLE
12282	/**
12283	* Handles a guest VM-exit from hardware-assisted VMX execution.
12284	*
12285	* @returns Strict VBox status code (i.e. informational status codes too).
12286	* @param pVCpu The cross context virtual CPU structure.
12287	* @param pVmxTransient The VMX-transient structure.
12288	*/
12289	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12290	{
12291	#ifdef DEBUG_ramshankar
12292	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12293	do { \
12294	if (a_fSave != 0) \
12295	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
12296	VBOXSTRICTRC rcStrict = a_CallExpr; \
12297	if (a_fSave != 0) \
12298	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12299	return rcStrict; \
12300	} while (0)
12301	#else
12302	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12303	#endif
12304	uint32_t const rcReason = pVmxTransient->uExitReason;
12305	switch (rcReason)
12306	{
12307	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12308	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12309	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12310	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12311	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12312	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12313	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12314	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12315	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12316	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12317	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12318	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12319	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12320	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12321	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12322	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12323	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12324	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12325	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12326	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12327	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12328	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12329	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12330	case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
12331	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12332	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12333	case VMX_EXIT_GDTR_IDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12334	case VMX_EXIT_LDTR_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12335	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12336	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12337	case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
12338	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12339	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12340	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12341	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12342	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12343	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12344	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12345	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12346	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12347	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12348	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12349	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12350	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12351	#else
12352	case VMX_EXIT_VMCLEAR:
12353	case VMX_EXIT_VMLAUNCH:
12354	case VMX_EXIT_VMPTRLD:
12355	case VMX_EXIT_VMPTRST:
12356	case VMX_EXIT_VMREAD:
12357	case VMX_EXIT_VMRESUME:
12358	case VMX_EXIT_VMWRITE:
12359	case VMX_EXIT_VMXOFF:
12360	case VMX_EXIT_VMXON:
12361	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12362	#endif
12363
12364	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12365	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12366	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
12367	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
12368	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
12369	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
12370	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
12371	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12372	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
12373
12374	case VMX_EXIT_INVEPT:
12375	case VMX_EXIT_INVVPID:
12376	case VMX_EXIT_VMFUNC:
12377	case VMX_EXIT_XSAVES:
12378	case VMX_EXIT_XRSTORS:
12379	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12380
12381	case VMX_EXIT_ENCLS:
12382	case VMX_EXIT_RDSEED:
12383	case VMX_EXIT_PML_FULL:
12384	default:
12385	return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
12386	}
12387	#undef VMEXIT_CALL_RET
12388	}
12389	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12390
12391
12392	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12393	/**
12394	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12395	*
12396	* @returns Strict VBox status code (i.e. informational status codes too).
12397	* @param pVCpu The cross context virtual CPU structure.
12398	* @param pVmxTransient The VMX-transient structure.
12399	*/
12400	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12401	{
12402	uint32_t const rcReason = pVmxTransient->uExitReason;
12403	switch (rcReason)
12404	{
12405	case VMX_EXIT_EPT_MISCONFIG:
12406	case VMX_EXIT_EPT_VIOLATION:
12407	case VMX_EXIT_IO_INSTR:
12408	case VMX_EXIT_CPUID:
12409	case VMX_EXIT_RDTSC:
12410	case VMX_EXIT_RDTSCP:
12411	case VMX_EXIT_APIC_ACCESS:
12412	case VMX_EXIT_XCPT_OR_NMI:
12413	case VMX_EXIT_MOV_CRX:
12414	case VMX_EXIT_EXT_INT:
12415	case VMX_EXIT_INT_WINDOW:
12416	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12417	case VMX_EXIT_MWAIT:
12418	case VMX_EXIT_MONITOR:
12419	case VMX_EXIT_TASK_SWITCH:
12420	case VMX_EXIT_PREEMPT_TIMER:
12421	case VMX_EXIT_RDMSR:
12422	case VMX_EXIT_WRMSR:
12423	case VMX_EXIT_VMCALL:
12424	case VMX_EXIT_MOV_DRX:
12425	case VMX_EXIT_HLT:
12426	case VMX_EXIT_INVD:
12427	case VMX_EXIT_INVLPG:
12428	case VMX_EXIT_RSM:
12429	case VMX_EXIT_MTF:
12430	case VMX_EXIT_PAUSE:
12431	case VMX_EXIT_GDTR_IDTR_ACCESS:
12432	case VMX_EXIT_LDTR_TR_ACCESS:
12433	case VMX_EXIT_WBINVD:
12434	case VMX_EXIT_XSETBV:
12435	case VMX_EXIT_RDRAND:
12436	case VMX_EXIT_INVPCID:
12437	case VMX_EXIT_GETSEC:
12438	case VMX_EXIT_RDPMC:
12439	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12440	case VMX_EXIT_VMCLEAR:
12441	case VMX_EXIT_VMLAUNCH:
12442	case VMX_EXIT_VMPTRLD:
12443	case VMX_EXIT_VMPTRST:
12444	case VMX_EXIT_VMREAD:
12445	case VMX_EXIT_VMRESUME:
12446	case VMX_EXIT_VMWRITE:
12447	case VMX_EXIT_VMXOFF:
12448	case VMX_EXIT_VMXON:
12449	#endif
12450	case VMX_EXIT_TRIPLE_FAULT:
12451	case VMX_EXIT_NMI_WINDOW:
12452	case VMX_EXIT_INIT_SIGNAL:
12453	case VMX_EXIT_SIPI:
12454	case VMX_EXIT_IO_SMI:
12455	case VMX_EXIT_SMI:
12456	case VMX_EXIT_ERR_MSR_LOAD:
12457	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12458	case VMX_EXIT_ERR_MACHINE_CHECK:
12459
12460	case VMX_EXIT_INVEPT:
12461	case VMX_EXIT_INVVPID:
12462	case VMX_EXIT_VMFUNC:
12463	case VMX_EXIT_XSAVES:
12464	case VMX_EXIT_XRSTORS:
12465
12466	case VMX_EXIT_ENCLS:
12467	case VMX_EXIT_RDSEED:
12468	case VMX_EXIT_PML_FULL:
12469	default:
12470	return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
12471	}
12472	#undef VMEXIT_CALL_RET
12473	}
12474	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12475
12476
12477	#ifdef VBOX_STRICT
12478	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
12479	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
12480	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
12481
12482	# define HMVMX_ASSERT_PREEMPT_CPUID() \
12483	do { \
12484	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
12485	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
12486	} while (0)
12487
12488	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12489	do { \
12490	AssertPtr((a_pVCpu)); \
12491	AssertPtr((a_pVmxTransient)); \
12492	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
12493	Assert((a_pVmxTransient)->pVmcsInfo); \
12494	Assert(ASMIntAreEnabled()); \
12495	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12496	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
12497	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
12498	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12499	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
12500	HMVMX_ASSERT_PREEMPT_CPUID(); \
12501	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12502	} while (0)
12503
12504	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12505	do { \
12506	Log4Func(("\n")); \
12507	} while (0)
12508	#else
12509	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12510	do { \
12511	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12512	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
12513	} while (0)
12514	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
12515	#endif
12516
12517
12518	/**
12519	* Advances the guest RIP by the specified number of bytes.
12520	*
12521	* @param pVCpu The cross context virtual CPU structure.
12522	* @param cbInstr Number of bytes to advance the RIP by.
12523	*
12524	* @remarks No-long-jump zone!!!
12525	*/
12526	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
12527	{
12528	/* Advance the RIP. */
12529	pVCpu->cpum.GstCtx.rip += cbInstr;
12530	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12531
12532	/* Update interrupt inhibition. */
12533	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
12534	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
12535	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12536	}
12537
12538
12539	/**
12540	* Advances the guest RIP after reading it from the VMCS.
12541	*
12542	* @returns VBox status code, no informational status codes.
12543	* @param pVCpu The cross context virtual CPU structure.
12544	* @param pVmxTransient The VMX-transient structure.
12545	*
12546	* @remarks No-long-jump zone!!!
12547	*/
12548	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12549	{
12550	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12551	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
12552	AssertRCReturn(rc, rc);
12553
12554	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
12555	return VINF_SUCCESS;
12556	}
12557
12558
12559	/**
12560	* Handle a condition that occurred while delivering an event through the guest
12561	* IDT.
12562	*
12563	* @returns Strict VBox status code (i.e. informational status codes too).
12564	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
12565	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
12566	* to continue execution of the guest which will delivery the \#DF.
12567	* @retval VINF_EM_RESET if we detected a triple-fault condition.
12568	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
12569	*
12570	* @param pVCpu The cross context virtual CPU structure.
12571	* @param pVmxTransient The VMX-transient structure.
12572	*
12573	* @remarks No-long-jump zone!!!
12574	*/
12575	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12576	{
12577	uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12578
12579	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12580	rc2 \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12581	AssertRCReturn(rc2, rc2);
12582
12583	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
12584	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12585	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12586	{
12587	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12588	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12589
12590	/*
12591	* If the event was a software interrupt (generated with INT n) or a software exception
12592	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
12593	* can handle the VM-exit and continue guest execution which will re-execute the
12594	* instruction rather than re-injecting the exception, as that can cause premature
12595	* trips to ring-3 before injection and involve TRPM which currently has no way of
12596	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
12597	* the problem).
12598	*/
12599	IEMXCPTRAISE enmRaise;
12600	IEMXCPTRAISEINFO fRaiseInfo;
12601	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
12602	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
12603	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
12604	{
12605	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
12606	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12607	}
12608	else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
12609	{
12610	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12611	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
12612	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
12613	/** @todo Make AssertMsgReturn as just AssertMsg later. */
12614	AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
12615	("Unexpected VM-exit interruption vector type %#x!\n", uExitVectorType), VERR_VMX_IPE_5);
12616
12617	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
12618
12619	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
12620	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
12621	{
12622	pVmxTransient->fVectoringPF = true;
12623	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12624	}
12625	}
12626	else
12627	{
12628	/*
12629	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
12630	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
12631	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
12632	*/
12633	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12634	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12635	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
12636	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12637	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12638	}
12639
12640	/*
12641	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
12642	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
12643	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
12644	* subsequent VM-entry would fail.
12645	*
12646	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
12647	*/
12648	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS)
12649	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12650	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
12651	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
12652	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
12653	{
12654	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
12655	}
12656
12657	switch (enmRaise)
12658	{
12659	case IEMXCPTRAISE_CURRENT_XCPT:
12660	{
12661	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
12662	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
12663	Assert(rcStrict == VINF_SUCCESS);
12664	break;
12665	}
12666
12667	case IEMXCPTRAISE_PREV_EVENT:
12668	{
12669	uint32_t u32ErrCode;
12670	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
12671	{
12672	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12673	AssertRCReturn(rc2, rc2);
12674	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12675	}
12676	else
12677	u32ErrCode = 0;
12678
12679	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
12680	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
12681	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12682	0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
12683
12684	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
12685	pVCpu->hm.s.Event.u32ErrCode));
12686	Assert(rcStrict == VINF_SUCCESS);
12687	break;
12688	}
12689
12690	case IEMXCPTRAISE_REEXEC_INSTR:
12691	Assert(rcStrict == VINF_SUCCESS);
12692	break;
12693
12694	case IEMXCPTRAISE_DOUBLE_FAULT:
12695	{
12696	/*
12697	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
12698	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
12699	*/
12700	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
12701	{
12702	pVmxTransient->fVectoringDoublePF = true;
12703	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
12704	pVCpu->cpum.GstCtx.cr2));
12705	rcStrict = VINF_SUCCESS;
12706	}
12707	else
12708	{
12709	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
12710	hmR0VmxSetPendingXcptDF(pVCpu);
12711	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
12712	uIdtVector, uExitVector));
12713	rcStrict = VINF_HM_DOUBLE_FAULT;
12714	}
12715	break;
12716	}
12717
12718	case IEMXCPTRAISE_TRIPLE_FAULT:
12719	{
12720	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
12721	rcStrict = VINF_EM_RESET;
12722	break;
12723	}
12724
12725	case IEMXCPTRAISE_CPU_HANG:
12726	{
12727	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
12728	rcStrict = VERR_EM_GUEST_CPU_HANG;
12729	break;
12730	}
12731
12732	default:
12733	{
12734	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
12735	rcStrict = VERR_VMX_IPE_2;
12736	break;
12737	}
12738	}
12739	}
12740	else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
12741	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
12742	&& uExitVector != X86_XCPT_DF
12743	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
12744	{
12745	/*
12746	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
12747	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
12748	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
12749	*/
12750	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
12751	{
12752	Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
12753	VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
12754	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
12755	}
12756	}
12757
12758	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
12759	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
12760	return rcStrict;
12761	}
12762
12763
12764	/** @name VM-exit handlers.
12765	* @{
12766	*/
12767	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12768	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12769	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12770
12771	/**
12772	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
12773	*/
12774	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12775	{
12776	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12777	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
12778	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
12779	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
12780	return VINF_SUCCESS;
12781	return VINF_EM_RAW_INTERRUPT;
12782	}
12783
12784
12785	/**
12786	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
12787	*/
12788	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12789	{
12790	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12791	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
12792
12793	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12794	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12795	AssertRCReturn(rc, rc);
12796
12797	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12798	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
12799	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
12800	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
12801
12802	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12803	{
12804	/*
12805	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
12806	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
12807	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
12808	*
12809	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
12810	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
12811	*/
12812	VMXDispatchHostNmi();
12813	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
12814	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12815	return VINF_SUCCESS;
12816	}
12817
12818	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12819	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12820	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
12821	{ /* likely */ }
12822	else
12823	{
12824	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
12825	rcStrictRc1 = VINF_SUCCESS;
12826	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12827	return rcStrictRc1;
12828	}
12829
12830	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
12831	uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
12832	switch (uIntType)
12833	{
12834	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
12835	Assert(uVector == X86_XCPT_DB);
12836	RT_FALL_THRU();
12837	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
12838	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
12839	RT_FALL_THRU();
12840	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
12841	{
12842	/*
12843	* If there's any exception caused as a result of event injection, the resulting
12844	* secondary/final execption will be pending, we shall continue guest execution
12845	* after injecting the event. The page-fault case is complicated and we manually
12846	* handle any currently pending event in hmR0VmxExitXcptPF.
12847	*/
12848	if (!pVCpu->hm.s.Event.fPending)
12849	{ /* likely */ }
12850	else if (uVector != X86_XCPT_PF)
12851	{
12852	rc = VINF_SUCCESS;
12853	break;
12854	}
12855
12856	switch (uVector)
12857	{
12858	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
12859	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
12860	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
12861	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
12862	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
12863	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
12864
12865	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
12866	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12867	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
12868	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12869	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
12870	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12871	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
12872	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12873	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
12874	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12875	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
12876	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12877	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
12878	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12879	default:
12880	{
12881	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
12882	if (pVmcsInfo->RealMode.fRealOnV86Active)
12883	{
12884	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
12885	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
12886	Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
12887
12888	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
12889	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12890	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12891	AssertRCReturn(rc, rc);
12892	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
12893	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
12894	0 /* GCPtrFaultAddress */);
12895	}
12896	else
12897	{
12898	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
12899	pVCpu->hm.s.u32HMError = uVector;
12900	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
12901	}
12902	break;
12903	}
12904	}
12905	break;
12906	}
12907
12908	default:
12909	{
12910	pVCpu->hm.s.u32HMError = uExitIntInfo;
12911	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
12912	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
12913	break;
12914	}
12915	}
12916	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12917	return rc;
12918	}
12919
12920
12921	/**
12922	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
12923	*/
12924	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12925	{
12926	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12927
12928	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
12929	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12930	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
12931	AssertRCReturn(rc, rc);
12932
12933	/* Evaluate and deliver pending events and resume guest execution. */
12934	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
12935	return VINF_SUCCESS;
12936	}
12937
12938
12939	/**
12940	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
12941	*/
12942	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12943	{
12944	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12945
12946	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12947	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
12948	{
12949	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
12950	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12951	}
12952
12953	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS));
12954
12955	/*
12956	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
12957	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
12958	*/
12959	uint32_t fIntrState;
12960	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
12961	AssertRCReturn(rc, rc);
12962	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
12963	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
12964	{
12965	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
12966	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12967
12968	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
12969	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
12970	AssertRCReturn(rc, rc);
12971	}
12972
12973	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
12974	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
12975	AssertRCReturn(rc, rc);
12976
12977	/* Evaluate and deliver pending events and resume guest execution. */
12978	return VINF_SUCCESS;
12979	}
12980
12981
12982	/**
12983	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
12984	*/
12985	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12986	{
12987	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12988	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12989	}
12990
12991
12992	/**
12993	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
12994	*/
12995	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12996	{
12997	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12998	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12999	}
13000
13001
13002	/**
13003	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
13004	*/
13005	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13006	{
13007	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13008
13009	/*
13010	* Get the state we need and update the exit history entry.
13011	*/
13012	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13013	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13014	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13015	AssertRCReturn(rc, rc);
13016
13017	VBOXSTRICTRC rcStrict;
13018	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
13019	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
13020	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
13021	if (!pExitRec)
13022	{
13023	/*
13024	* Regular CPUID instruction execution.
13025	*/
13026	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
13027	if (rcStrict == VINF_SUCCESS)
13028	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13029	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13030	{
13031	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13032	rcStrict = VINF_SUCCESS;
13033	}
13034	}
13035	else
13036	{
13037	/*
13038	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
13039	*/
13040	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13041	AssertRCReturn(rc2, rc2);
13042
13043	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
13044	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
13045
13046	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
13047	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13048
13049	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
13050	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13051	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
13052	}
13053	return rcStrict;
13054	}
13055
13056
13057	/**
13058	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
13059	*/
13060	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13061	{
13062	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13063
13064	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13065	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
13066	AssertRCReturn(rc, rc);
13067
13068	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
13069	return VINF_EM_RAW_EMULATE_INSTR;
13070
13071	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
13072	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13073	}
13074
13075
13076	/**
13077	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
13078	*/
13079	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13080	{
13081	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13082
13083	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13084	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13085	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13086	AssertRCReturn(rc, rc);
13087
13088	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
13089	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13090	{
13091	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13092	we must reset offsetting on VM-entry. See @bugref{6634}. */
13093	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13094	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13095	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13096	}
13097	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13098	{
13099	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13100	rcStrict = VINF_SUCCESS;
13101	}
13102	return rcStrict;
13103	}
13104
13105
13106	/**
13107	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
13108	*/
13109	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13110	{
13111	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13112
13113	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13114	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
13115	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13116	AssertRCReturn(rc, rc);
13117
13118	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
13119	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13120	{
13121	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13122	we must reset offsetting on VM-reentry. See @bugref{6634}. */
13123	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13124	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13125	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13126	}
13127	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13128	{
13129	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13130	rcStrict = VINF_SUCCESS;
13131	}
13132	return rcStrict;
13133	}
13134
13135
13136	/**
13137	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
13138	*/
13139	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13140	{
13141	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13142
13143	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13144	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
13145	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13146	AssertRCReturn(rc, rc);
13147
13148	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13149	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13150	if (RT_LIKELY(rc == VINF_SUCCESS))
13151	{
13152	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13153	Assert(pVmxTransient->cbInstr == 2);
13154	}
13155	else
13156	{
13157	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
13158	rc = VERR_EM_INTERPRETER;
13159	}
13160	return rc;
13161	}
13162
13163
13164	/**
13165	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
13166	*/
13167	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13168	{
13169	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13170
13171	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
13172	if (EMAreHypercallInstructionsEnabled(pVCpu))
13173	{
13174	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13175	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
13176	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13177	AssertRCReturn(rc, rc);
13178
13179	/* Perform the hypercall. */
13180	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
13181	if (rcStrict == VINF_SUCCESS)
13182	{
13183	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13184	AssertRCReturn(rc, rc);
13185	}
13186	else
13187	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
13188	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
13189	\|\| RT_FAILURE(rcStrict));
13190
13191	/* If the hypercall changes anything other than guest's general-purpose registers,
13192	we would need to reload the guest changed bits here before VM-entry. */
13193	}
13194	else
13195	Log4Func(("Hypercalls not enabled\n"));
13196
13197	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
13198	if (RT_FAILURE(rcStrict))
13199	{
13200	hmR0VmxSetPendingXcptUD(pVCpu);
13201	rcStrict = VINF_SUCCESS;
13202	}
13203
13204	return rcStrict;
13205	}
13206
13207
13208	/**
13209	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
13210	*/
13211	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13212	{
13213	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13214	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
13215
13216	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13217	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13218	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13219	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13220	AssertRCReturn(rc, rc);
13221
13222	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
13223
13224	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
13225	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13226	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13227	{
13228	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13229	rcStrict = VINF_SUCCESS;
13230	}
13231	else
13232	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n", pVmxTransient->uExitQual,
13233	VBOXSTRICTRC_VAL(rcStrict)));
13234	return rcStrict;
13235	}
13236
13237
13238	/**
13239	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
13240	*/
13241	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13242	{
13243	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13244
13245	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13246	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13247	AssertRCReturn(rc, rc);
13248
13249	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13250	rc = EMInterpretMonitor(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13251	if (RT_LIKELY(rc == VINF_SUCCESS))
13252	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13253	else
13254	{
13255	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
13256	rc = VERR_EM_INTERPRETER;
13257	}
13258	return rc;
13259	}
13260
13261
13262	/**
13263	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
13264	*/
13265	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13266	{
13267	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13268
13269	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13270	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13271	AssertRCReturn(rc, rc);
13272
13273	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13274	VBOXSTRICTRC rc2 = EMInterpretMWait(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13275	rc = VBOXSTRICTRC_VAL(rc2);
13276	if (RT_LIKELY( rc == VINF_SUCCESS
13277	\|\| rc == VINF_EM_HALT))
13278	{
13279	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13280	AssertRCReturn(rc3, rc3);
13281
13282	if ( rc == VINF_EM_HALT
13283	&& EMMonitorWaitShouldContinue(pVCpu, pCtx))
13284	rc = VINF_SUCCESS;
13285	}
13286	else
13287	{
13288	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
13289	rc = VERR_EM_INTERPRETER;
13290	}
13291	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
13292	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
13293	return rc;
13294	}
13295
13296
13297	/**
13298	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
13299	*/
13300	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13301	{
13302	/*
13303	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
13304	* mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
13305	* treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
13306	* VMX root operation. If we get here, something funny is going on.
13307	*
13308	* See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
13309	*/
13310	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13311	AssertMsgFailed(("Unexpected RSM VM-exit\n"));
13312	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13313	}
13314
13315
13316	/**
13317	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
13318	*/
13319	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13320	{
13321	/*
13322	* This can only happen if we support dual-monitor treatment of SMI, which can be activated
13323	* by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
13324	* this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
13325	* an SMI. If we get here, something funny is going on.
13326	*
13327	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
13328	* See Intel spec. 25.3 "Other Causes of VM-Exits"
13329	*/
13330	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13331	AssertMsgFailed(("Unexpected SMI VM-exit\n"));
13332	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13333	}
13334
13335
13336	/**
13337	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
13338	*/
13339	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13340	{
13341	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
13342	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13343	AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
13344	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13345	}
13346
13347
13348	/**
13349	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
13350	*/
13351	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13352	{
13353	/*
13354	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
13355	* We don't make use of it as our guests don't have direct access to the host LAPIC.
13356	* See Intel spec. 25.3 "Other Causes of VM-exits".
13357	*/
13358	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13359	AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
13360	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13361	}
13362
13363
13364	/**
13365	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
13366	* VM-exit.
13367	*/
13368	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13369	{
13370	/*
13371	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
13372	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
13373	*
13374	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
13375	* See Intel spec. "23.8 Restrictions on VMX operation".
13376	*/
13377	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13378	return VINF_SUCCESS;
13379	}
13380
13381
13382	/**
13383	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
13384	* VM-exit.
13385	*/
13386	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13387	{
13388	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13389	return VINF_EM_RESET;
13390	}
13391
13392
13393	/**
13394	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
13395	*/
13396	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13397	{
13398	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13399
13400	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13401	AssertRCReturn(rc, rc);
13402
13403	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
13404	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
13405	rc = VINF_SUCCESS;
13406	else
13407	rc = VINF_EM_HALT;
13408
13409	if (rc != VINF_SUCCESS)
13410	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
13411	return rc;
13412	}
13413
13414
13415	/**
13416	* VM-exit handler for instructions that result in a \#UD exception delivered to
13417	* the guest.
13418	*/
13419	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13420	{
13421	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13422	hmR0VmxSetPendingXcptUD(pVCpu);
13423	return VINF_SUCCESS;
13424	}
13425
13426
13427	/**
13428	* VM-exit handler for expiry of the VMX-preemption timer.
13429	*/
13430	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13431	{
13432	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13433
13434	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
13435	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13436
13437	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
13438	PVM pVM = pVCpu->CTX_SUFF(pVM);
13439	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
13440	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
13441	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
13442	}
13443
13444
13445	/**
13446	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
13447	*/
13448	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13449	{
13450	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13451
13452	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13453	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13454	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
13455	AssertRCReturn(rc, rc);
13456
13457	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
13458	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13459	: HM_CHANGED_RAISED_XCPT_MASK);
13460
13461	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13462	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
13463
13464	return rcStrict;
13465	}
13466
13467
13468	/**
13469	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
13470	*/
13471	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13472	{
13473	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13474	/** @todo Use VM-exit instruction information. */
13475	return VERR_EM_INTERPRETER;
13476	}
13477
13478
13479	/**
13480	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
13481	* Error VM-exit.
13482	*/
13483	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13484	{
13485	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13486	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13487	AssertRCReturn(rc, rc);
13488
13489	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo);
13490	if (RT_FAILURE(rc))
13491	return rc;
13492
13493	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
13494	NOREF(uInvalidReason);
13495
13496	#ifdef VBOX_STRICT
13497	uint32_t fIntrState;
13498	RTHCUINTREG uHCReg;
13499	uint64_t u64Val;
13500	uint32_t u32Val;
13501	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
13502	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
13503	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
13504	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13505	AssertRCReturn(rc, rc);
13506
13507	Log4(("uInvalidReason %u\n", uInvalidReason));
13508	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
13509	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
13510	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
13511	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
13512
13513	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
13514	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
13515	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
13516	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
13517	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
13518	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13519	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
13520	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
13521	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
13522	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13523	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
13524	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
13525
13526	hmR0DumpRegs(pVCpu);
13527	#endif
13528
13529	return VERR_VMX_INVALID_GUEST_STATE;
13530	}
13531
13532
13533	/**
13534	* VM-exit handler for VM-entry failure due to an MSR-load
13535	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
13536	*/
13537	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13538	{
13539	AssertMsgFailed(("Unexpected MSR-load exit\n"));
13540	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13541	}
13542
13543
13544	/**
13545	* VM-exit handler for VM-entry failure due to a machine-check event
13546	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
13547	*/
13548	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13549	{
13550	AssertMsgFailed(("Unexpected machine-check event exit\n"));
13551	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13552	}
13553
13554
13555	/**
13556	* VM-exit handler for all undefined reasons. Should never ever happen.. in
13557	* theory.
13558	*/
13559	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13560	{
13561	RT_NOREF2(pVCpu, pVmxTransient);
13562	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
13563	return VERR_VMX_UNDEFINED_EXIT_CODE;
13564	}
13565
13566
13567	/**
13568	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
13569	* (VMX_EXIT_GDTR_IDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
13570	* Conditional VM-exit.
13571	*/
13572	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13573	{
13574	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13575
13576	/* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
13577	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
13578	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13579	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
13580	return VERR_EM_INTERPRETER;
13581	AssertMsgFailed(("Unexpected XDTR access\n"));
13582	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13583	}
13584
13585
13586	/**
13587	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
13588	*/
13589	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13590	{
13591	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13592
13593	/* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
13594	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13595	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
13596	return VERR_EM_INTERPRETER;
13597	AssertMsgFailed(("Unexpected RDRAND exit\n"));
13598	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13599	}
13600
13601
13602	/**
13603	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
13604	*/
13605	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13606	{
13607	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13608
13609	/** @todo Optimize this: We currently drag in in the whole MSR state
13610	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13611	* MSRs required. That would require changes to IEM and possibly CPUM too.
13612	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13613	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13614	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13615	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13616	switch (idMsr)
13617	{
13618	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13619	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13620	}
13621
13622	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13623	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13624	AssertRCReturn(rc, rc);
13625
13626	Log4Func(("ecx=%#RX32\n", idMsr));
13627
13628	#ifdef VBOX_STRICT
13629	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
13630	{
13631	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
13632	&& idMsr != MSR_K6_EFER)
13633	{
13634	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
13635	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13636	}
13637	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13638	{
13639	Assert(pVmcsInfo->pvMsrBitmap);
13640	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13641	if (fMsrpm & VMXMSRPM_ALLOW_RD)
13642	{
13643	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
13644	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13645	}
13646	}
13647	}
13648	#endif
13649
13650	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
13651	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
13652	if (rcStrict == VINF_SUCCESS)
13653	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13654	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
13655	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13656	{
13657	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13658	rcStrict = VINF_SUCCESS;
13659	}
13660	else
13661	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13662
13663	return rcStrict;
13664	}
13665
13666
13667	/**
13668	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
13669	*/
13670	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13671	{
13672	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13673
13674	/** @todo Optimize this: We currently drag in in the whole MSR state
13675	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13676	* MSRs required. That would require changes to IEM and possibly CPUM too.
13677	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13678	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13679	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13680
13681	/*
13682	* The FS and GS base MSRs are not part of the above all-MSRs mask.
13683	* Although we don't need to fetch the base as it will be overwritten shortly, while
13684	* loading guest-state we would also load the entire segment register including limit
13685	* and attributes and thus we need to load them here.
13686	*/
13687	switch (idMsr)
13688	{
13689	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13690	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13691	}
13692
13693	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13694	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13695	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13696	AssertRCReturn(rc, rc);
13697
13698	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
13699
13700	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
13701	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
13702
13703	if (rcStrict == VINF_SUCCESS)
13704	{
13705	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13706
13707	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
13708	if ( idMsr == MSR_IA32_APICBASE
13709	\|\| ( idMsr >= MSR_IA32_X2APIC_START
13710	&& idMsr <= MSR_IA32_X2APIC_END))
13711	{
13712	/*
13713	* We've already saved the APIC related guest-state (TPR) in post-run phase.
13714	* When full APIC register virtualization is implemented we'll have to make
13715	* sure APIC state is saved from the VMCS before IEM changes it.
13716	*/
13717	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
13718	}
13719	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
13720	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13721	else if (idMsr == MSR_K6_EFER)
13722	{
13723	/*
13724	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
13725	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
13726	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
13727	*/
13728	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
13729	}
13730
13731	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
13732	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
13733	{
13734	switch (idMsr)
13735	{
13736	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
13737	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
13738	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
13739	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
13740	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
13741	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
13742	default:
13743	{
13744	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
13745	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
13746	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13747	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
13748	break;
13749	}
13750	}
13751	}
13752	#ifdef VBOX_STRICT
13753	else
13754	{
13755	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
13756	switch (idMsr)
13757	{
13758	case MSR_IA32_SYSENTER_CS:
13759	case MSR_IA32_SYSENTER_EIP:
13760	case MSR_IA32_SYSENTER_ESP:
13761	case MSR_K8_FS_BASE:
13762	case MSR_K8_GS_BASE:
13763	{
13764	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
13765	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13766	}
13767
13768	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
13769	default:
13770	{
13771	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
13772	{
13773	/* EFER MSR writes are always intercepted. */
13774	if (idMsr != MSR_K6_EFER)
13775	{
13776	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
13777	idMsr));
13778	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13779	}
13780	}
13781
13782	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13783	{
13784	Assert(pVmcsInfo->pvMsrBitmap);
13785	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13786	if (fMsrpm & VMXMSRPM_ALLOW_WR)
13787	{
13788	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
13789	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13790	}
13791	}
13792	break;
13793	}
13794	}
13795	}
13796	#endif /* VBOX_STRICT */
13797	}
13798	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13799	{
13800	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13801	rcStrict = VINF_SUCCESS;
13802	}
13803	else
13804	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13805
13806	return rcStrict;
13807	}
13808
13809
13810	/**
13811	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
13812	*/
13813	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13814	{
13815	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13816	/** @todo The guest has likely hit a contended spinlock. We might want to
13817	* poke a schedule different guest VCPU. */
13818	return VINF_EM_RAW_INTERRUPT;
13819	}
13820
13821
13822	/**
13823	* VM-exit handler for when the TPR value is lowered below the specified
13824	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
13825	*/
13826	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13827	{
13828	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13829	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
13830
13831	/*
13832	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
13833	* We'll re-evaluate pending interrupts and inject them before the next VM
13834	* entry so we can just continue execution here.
13835	*/
13836	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
13837	return VINF_SUCCESS;
13838	}
13839
13840
13841	/**
13842	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
13843	* VM-exit.
13844	*
13845	* @retval VINF_SUCCESS when guest execution can continue.
13846	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
13847	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
13848	* interpreter.
13849	*/
13850	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13851	{
13852	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13853	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
13854
13855	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13856	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13857	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13858	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13859	AssertRCReturn(rc, rc);
13860
13861	VBOXSTRICTRC rcStrict;
13862	PVM pVM = pVCpu->CTX_SUFF(pVM);
13863	RTGCUINTPTR const uExitQual = pVmxTransient->uExitQual;
13864	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
13865	switch (uAccessType)
13866	{
13867	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
13868	{
13869	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
13870	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
13871	VMX_EXIT_QUAL_CRX_GENREG(uExitQual));
13872	AssertMsg( rcStrict == VINF_SUCCESS
13873	\|\| rcStrict == VINF_IEM_RAISED_XCPT
13874	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13875
13876	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
13877	{
13878	case 0:
13879	{
13880	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13881	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13882	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
13883	Log4Func(("CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
13884
13885	/*
13886	* This is a kludge for handling switches back to real mode when we try to use
13887	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
13888	* deal with special selector values, so we have to return to ring-3 and run
13889	* there till the selector values are V86 mode compatible.
13890	*
13891	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
13892	* latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
13893	* at the end of this function.
13894	*/
13895	if ( rc == VINF_SUCCESS
13896	&& !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
13897	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
13898	&& (uOldCr0 & X86_CR0_PE)
13899	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
13900	{
13901	/** @todo check selectors rather than returning all the time. */
13902	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
13903	rcStrict = VINF_EM_RESCHEDULE_REM;
13904	}
13905	break;
13906	}
13907
13908	case 2:
13909	{
13910	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
13911	/* Nothing to do here, CR2 it's not part of the VMCS. */
13912	break;
13913	}
13914
13915	case 3:
13916	{
13917	Assert( !pVM->hm.s.fNestedPaging
13918	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
13919	\|\| pVCpu->hm.s.fUsingDebugLoop);
13920	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
13921	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13922	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
13923	Log4Func(("CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
13924	break;
13925	}
13926
13927	case 4:
13928	{
13929	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
13930	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13931	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
13932	Log4Func(("CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
13933	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
13934	break;
13935	}
13936
13937	case 8:
13938	{
13939	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
13940	Assert(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
13941	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13942	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
13943	break;
13944	}
13945	default:
13946	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual)));
13947	break;
13948	}
13949	break;
13950	}
13951
13952	case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
13953	{
13954	Assert( !pVM->hm.s.fNestedPaging
13955	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
13956	\|\| pVCpu->hm.s.fUsingDebugLoop
13957	\|\| VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 3);
13958	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
13959	Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 8
13960	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
13961
13962	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_GENREG(uExitQual),
13963	VMX_EXIT_QUAL_CRX_REGISTER(uExitQual));
13964	AssertMsg( rcStrict == VINF_SUCCESS
13965	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13966	#ifdef VBOX_WITH_STATISTICS
13967	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
13968	{
13969	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
13970	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
13971	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
13972	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
13973	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
13974	}
13975	#endif
13976	Log4Func(("CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
13977	VBOXSTRICTRC_VAL(rcStrict)));
13978	if (VMX_EXIT_QUAL_CRX_GENREG(uExitQual) == X86_GREG_xSP)
13979	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
13980	else
13981	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13982	break;
13983	}
13984
13985	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
13986	{
13987	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
13988	AssertMsg( rcStrict == VINF_SUCCESS
13989	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13990
13991	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13992	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
13993	Log4Func(("CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
13994	break;
13995	}
13996
13997	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
13998	{
13999	/* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
14000	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
14001	AssertRCReturn(rc, rc);
14002	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual),
14003	pVmxTransient->uGuestLinearAddr);
14004	AssertMsg( rcStrict == VINF_SUCCESS
14005	\|\| rcStrict == VINF_IEM_RAISED_XCPT
14006	, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14007
14008	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14009	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
14010	Log4Func(("LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
14011	break;
14012	}
14013
14014	default:
14015	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
14016	VERR_VMX_UNEXPECTED_EXCEPTION);
14017	}
14018
14019	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
14020	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
14021	if (rcStrict == VINF_IEM_RAISED_XCPT)
14022	{
14023	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14024	rcStrict = VINF_SUCCESS;
14025	}
14026
14027	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
14028	NOREF(pVM);
14029	return rcStrict;
14030	}
14031
14032
14033	/**
14034	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
14035	* VM-exit.
14036	*/
14037	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14038	{
14039	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14040	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
14041
14042	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14043	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14044	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14045	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14046	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
14047	\| CPUMCTX_EXTRN_EFER);
14048	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
14049	AssertRCReturn(rc, rc);
14050
14051	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
14052	uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
14053	uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQual);
14054	bool fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
14055	bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
14056	bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
14057	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
14058	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
14059
14060	/*
14061	* Update exit history to see if this exit can be optimized.
14062	*/
14063	VBOXSTRICTRC rcStrict;
14064	PCEMEXITREC pExitRec = NULL;
14065	if ( !fGstStepping
14066	&& !fDbgStepping)
14067	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14068	!fIOString
14069	? !fIOWrite
14070	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
14071	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
14072	: !fIOWrite
14073	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
14074	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
14075	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14076	if (!pExitRec)
14077	{
14078	/* I/O operation lookup arrays. */
14079	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
14080	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
14081	uint32_t const cbValue = s_aIOSizes[uIOWidth];
14082	uint32_t const cbInstr = pVmxTransient->cbInstr;
14083	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
14084	PVM pVM = pVCpu->CTX_SUFF(pVM);
14085	if (fIOString)
14086	{
14087	/*
14088	* INS/OUTS - I/O String instruction.
14089	*
14090	* Use instruction-information if available, otherwise fall back on
14091	* interpreting the instruction.
14092	*/
14093	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14094	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
14095	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
14096	if (fInsOutsInfo)
14097	{
14098	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14099	AssertRCReturn(rc2, rc2);
14100	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
14101	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
14102	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
14103	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
14104	if (fIOWrite)
14105	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
14106	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
14107	else
14108	{
14109	/*
14110	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
14111	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
14112	* See Intel Instruction spec. for "INS".
14113	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
14114	*/
14115	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
14116	}
14117	}
14118	else
14119	rcStrict = IEMExecOne(pVCpu);
14120
14121	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14122	fUpdateRipAlready = true;
14123	}
14124	else
14125	{
14126	/*
14127	* IN/OUT - I/O instruction.
14128	*/
14129	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14130	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
14131	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
14132	if (fIOWrite)
14133	{
14134	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
14135	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
14136	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14137	&& !pCtx->eflags.Bits.u1TF)
14138	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
14139	}
14140	else
14141	{
14142	uint32_t u32Result = 0;
14143	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
14144	if (IOM_SUCCESS(rcStrict))
14145	{
14146	/* Save result of I/O IN instr. in AL/AX/EAX. */
14147	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
14148	}
14149	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14150	&& !pCtx->eflags.Bits.u1TF)
14151	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
14152	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
14153	}
14154	}
14155
14156	if (IOM_SUCCESS(rcStrict))
14157	{
14158	if (!fUpdateRipAlready)
14159	{
14160	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
14161	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14162	}
14163
14164	/*
14165	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
14166	* while booting Fedora 17 64-bit guest.
14167	*
14168	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
14169	*/
14170	if (fIOString)
14171	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
14172
14173	/*
14174	* If any I/O breakpoints are armed, we need to check if one triggered
14175	* and take appropriate action.
14176	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
14177	*/
14178	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
14179	AssertRCReturn(rc, rc);
14180
14181	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
14182	* execution engines about whether hyper BPs and such are pending. */
14183	uint32_t const uDr7 = pCtx->dr[7];
14184	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
14185	&& X86_DR7_ANY_RW_IO(uDr7)
14186	&& (pCtx->cr4 & X86_CR4_DE))
14187	\|\| DBGFBpIsHwIoArmed(pVM)))
14188	{
14189	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
14190
14191	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
14192	VMMRZCallRing3Disable(pVCpu);
14193	HM_DISABLE_PREEMPT(pVCpu);
14194
14195	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
14196
14197	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
14198	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
14199	{
14200	/* Raise #DB. */
14201	if (fIsGuestDbgActive)
14202	ASMSetDR6(pCtx->dr[6]);
14203	if (pCtx->dr[7] != uDr7)
14204	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
14205
14206	hmR0VmxSetPendingXcptDB(pVCpu);
14207	}
14208	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
14209	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
14210	else if ( rcStrict2 != VINF_SUCCESS
14211	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
14212	rcStrict = rcStrict2;
14213	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
14214
14215	HM_RESTORE_PREEMPT();
14216	VMMRZCallRing3Enable(pVCpu);
14217	}
14218	}
14219
14220	#ifdef VBOX_STRICT
14221	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14222	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
14223	Assert(!fIOWrite);
14224	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14225	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
14226	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
14227	Assert(fIOWrite);
14228	else
14229	{
14230	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
14231	* statuses, that the VMM device and some others may return. See
14232	* IOM_SUCCESS() for guidance. */
14233	AssertMsg( RT_FAILURE(rcStrict)
14234	\|\| rcStrict == VINF_SUCCESS
14235	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
14236	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
14237	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
14238	\|\| rcStrict == VINF_EM_RAW_TO_R3
14239	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14240	# endif
14241	}
14242	#endif
14243	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
14244	}
14245	else
14246	{
14247	/*
14248	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14249	*/
14250	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14251	AssertRCReturn(rc2, rc2);
14252	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
14253	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
14254	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
14255	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14256	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
14257	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
14258
14259	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14260	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14261
14262	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14263	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14264	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14265	}
14266	return rcStrict;
14267	}
14268
14269
14270	/**
14271	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
14272	* VM-exit.
14273	*/
14274	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14275	{
14276	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14277
14278	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
14279	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14280	AssertRCReturn(rc, rc);
14281	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
14282	{
14283	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14284	AssertRCReturn(rc, rc);
14285	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
14286	{
14287	uint32_t uErrCode;
14288	RTGCUINTPTR GCPtrFaultAddress;
14289	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
14290	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
14291	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
14292	if (fErrorCodeValid)
14293	{
14294	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14295	AssertRCReturn(rc, rc);
14296	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
14297	}
14298	else
14299	uErrCode = 0;
14300
14301	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
14302	&& uVector == X86_XCPT_PF)
14303	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
14304	else
14305	GCPtrFaultAddress = 0;
14306
14307	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14308	AssertRCReturn(rc, rc);
14309
14310	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
14311	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
14312
14313	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
14314	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14315	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14316	}
14317	}
14318
14319	/* Fall back to the interpreter to emulate the task-switch. */
14320	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14321	return VERR_EM_INTERPRETER;
14322	}
14323
14324
14325	/**
14326	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
14327	*/
14328	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14329	{
14330	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14331
14332	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14333	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
14334	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14335	AssertRCReturn(rc, rc);
14336	return VINF_EM_DBG_STEPPED;
14337	}
14338
14339
14340	/**
14341	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
14342	*/
14343	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14344	{
14345	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14346	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
14347
14348	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14349	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14350	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14351	{
14352	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
14353	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14354	{
14355	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14356	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14357	}
14358	}
14359	else
14360	{
14361	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14362	rcStrict1 = VINF_SUCCESS;
14363	return rcStrict1;
14364	}
14365
14366	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
14367	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14368	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14369	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14370	AssertRCReturn(rc, rc);
14371
14372	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
14373	uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
14374	VBOXSTRICTRC rcStrict2;
14375	switch (uAccessType)
14376	{
14377	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
14378	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
14379	{
14380	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
14381	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
14382	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
14383
14384	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
14385	GCPhys &= PAGE_BASE_GC_MASK;
14386	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
14387	PVM pVM = pVCpu->CTX_SUFF(pVM);
14388	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
14389	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
14390
14391	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14392	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
14393	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
14394	CPUMCTX2CORE(pCtx), GCPhys);
14395	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14396	if ( rcStrict2 == VINF_SUCCESS
14397	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14398	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14399	{
14400	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14401	\| HM_CHANGED_GUEST_APIC_TPR);
14402	rcStrict2 = VINF_SUCCESS;
14403	}
14404	break;
14405	}
14406
14407	default:
14408	Log4Func(("uAccessType=%#x\n", uAccessType));
14409	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
14410	break;
14411	}
14412
14413	if (rcStrict2 != VINF_SUCCESS)
14414	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
14415	return rcStrict2;
14416	}
14417
14418
14419	/**
14420	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
14421	* VM-exit.
14422	*/
14423	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14424	{
14425	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14426
14427	/* We should -not- get this VM-exit if the guest's debug registers were active. */
14428	if (pVmxTransient->fWasGuestDebugStateActive)
14429	{
14430	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
14431	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14432	}
14433
14434	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14435	if ( !pVCpu->hm.s.fSingleInstruction
14436	&& !pVmxTransient->fWasHyperDebugStateActive)
14437	{
14438	Assert(!DBGFIsStepping(pVCpu));
14439	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
14440
14441	/* Don't intercept MOV DRx any more. */
14442	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
14443	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14444	AssertRCReturn(rc, rc);
14445
14446	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
14447	VMMRZCallRing3Disable(pVCpu);
14448	HM_DISABLE_PREEMPT(pVCpu);
14449
14450	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
14451	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
14452	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
14453
14454	HM_RESTORE_PREEMPT();
14455	VMMRZCallRing3Enable(pVCpu);
14456
14457	#ifdef VBOX_WITH_STATISTICS
14458	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14459	AssertRCReturn(rc, rc);
14460	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14461	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14462	else
14463	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14464	#endif
14465	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
14466	return VINF_SUCCESS;
14467	}
14468
14469	/*
14470	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
14471	* The EFER MSR is always up-to-date.
14472	* Update the segment registers and DR7 from the CPU.
14473	*/
14474	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14475	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14476	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
14477	AssertRCReturn(rc, rc);
14478	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
14479
14480	PVM pVM = pVCpu->CTX_SUFF(pVM);
14481	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14482	{
14483	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14484	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
14485	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
14486	if (RT_SUCCESS(rc))
14487	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
14488	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14489	}
14490	else
14491	{
14492	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14493	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
14494	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
14495	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14496	}
14497
14498	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
14499	if (RT_SUCCESS(rc))
14500	{
14501	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14502	AssertRCReturn(rc2, rc2);
14503	return VINF_SUCCESS;
14504	}
14505	return rc;
14506	}
14507
14508
14509	/**
14510	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
14511	* Conditional VM-exit.
14512	*/
14513	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14514	{
14515	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14516	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14517
14518	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14519	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14520	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14521	{
14522	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
14523	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
14524	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14525	{
14526	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14527	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14528	}
14529	}
14530	else
14531	{
14532	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14533	rcStrict1 = VINF_SUCCESS;
14534	return rcStrict1;
14535	}
14536
14537	/*
14538	* Get sufficent state and update the exit history entry.
14539	*/
14540	RTGCPHYS GCPhys;
14541	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14542	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14543	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14544	AssertRCReturn(rc, rc);
14545
14546	VBOXSTRICTRC rcStrict;
14547	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14548	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
14549	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14550	if (!pExitRec)
14551	{
14552	/*
14553	* If we succeed, resume guest execution.
14554	* If we fail in interpreting the instruction because we couldn't get the guest physical address
14555	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
14556	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
14557	* weird case. See @bugref{6043}.
14558	*/
14559	PVM pVM = pVCpu->CTX_SUFF(pVM);
14560	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14561	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
14562	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
14563	if ( rcStrict == VINF_SUCCESS
14564	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
14565	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
14566	{
14567	/* Successfully handled MMIO operation. */
14568	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14569	\| HM_CHANGED_GUEST_APIC_TPR);
14570	rcStrict = VINF_SUCCESS;
14571	}
14572	}
14573	else
14574	{
14575	/*
14576	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14577	*/
14578	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14579	AssertRCReturn(rc2, rc2);
14580
14581	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
14582	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
14583
14584	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14585	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14586
14587	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14588	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14589	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14590	}
14591	return VBOXSTRICTRC_TODO(rcStrict);
14592	}
14593
14594
14595	/**
14596	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
14597	* VM-exit.
14598	*/
14599	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14600	{
14601	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14602	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14603
14604	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14605	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14606	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14607	{
14608	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
14609	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14610	Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
14611	}
14612	else
14613	{
14614	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14615	rcStrict1 = VINF_SUCCESS;
14616	return rcStrict1;
14617	}
14618
14619	RTGCPHYS GCPhys;
14620	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14621	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14622	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14623	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14624	AssertRCReturn(rc, rc);
14625
14626	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
14627	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
14628
14629	RTGCUINT uErrorCode = 0;
14630	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
14631	uErrorCode \|= X86_TRAP_PF_ID;
14632	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
14633	uErrorCode \|= X86_TRAP_PF_RW;
14634	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
14635	uErrorCode \|= X86_TRAP_PF_P;
14636
14637	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
14638
14639
14640	/* Handle the pagefault trap for the nested shadow table. */
14641	PVM pVM = pVCpu->CTX_SUFF(pVM);
14642	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14643
14644	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
14645	pCtx->cs.Sel, pCtx->rip));
14646
14647	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
14648	TRPMResetTrap(pVCpu);
14649
14650	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
14651	if ( rcStrict2 == VINF_SUCCESS
14652	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14653	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14654	{
14655	/* Successfully synced our nested page tables. */
14656	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
14657	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
14658	return VINF_SUCCESS;
14659	}
14660
14661	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14662	return rcStrict2;
14663	}
14664
14665	/** @} */
14666
14667	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14668	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14669	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14670
14671	/**
14672	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
14673	*/
14674	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14675	{
14676	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14677	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
14678
14679	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
14680	AssertRCReturn(rc, rc);
14681
14682	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
14683	{
14684	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
14685	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
14686
14687	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
14688	* provides VM-exit instruction length. If this causes problem later,
14689	* disassemble the instruction like it's done on AMD-V. */
14690	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14691	AssertRCReturn(rc2, rc2);
14692	return rc;
14693	}
14694
14695	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14696	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14697	return rc;
14698	}
14699
14700
14701	/**
14702	* VM-exit exception handler for \#BP (Breakpoint exception).
14703	*/
14704	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14705	{
14706	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14707	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
14708
14709	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14710	AssertRCReturn(rc, rc);
14711
14712	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14713	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14714	if (rc == VINF_EM_RAW_GUEST_TRAP)
14715	{
14716	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14717	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14718	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14719	AssertRCReturn(rc, rc);
14720
14721	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14722	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14723	}
14724
14725	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
14726	return rc;
14727	}
14728
14729
14730	/**
14731	* VM-exit exception handler for \#AC (alignment check exception).
14732	*/
14733	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14734	{
14735	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14736
14737	/*
14738	* Re-inject it. We'll detect any nesting before getting here.
14739	*/
14740	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14741	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14742	AssertRCReturn(rc, rc);
14743	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
14744
14745	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14746	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14747	return VINF_SUCCESS;
14748	}
14749
14750
14751	/**
14752	* VM-exit exception handler for \#DB (Debug exception).
14753	*/
14754	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14755	{
14756	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14757	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
14758
14759	/*
14760	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
14761	* for processing.
14762	*/
14763	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14764
14765	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
14766	uint64_t const uDR6 = X86_DR6_INIT_VAL
14767	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
14768	\| X86_DR6_BD \| X86_DR6_BS));
14769
14770	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14771	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
14772	Log6Func(("rc=%Rrc\n", rc));
14773	if (rc == VINF_EM_RAW_GUEST_TRAP)
14774	{
14775	/*
14776	* The exception was for the guest. Update DR6, DR7.GD and
14777	* IA32_DEBUGCTL.LBR before forwarding it.
14778	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
14779	*/
14780	VMMRZCallRing3Disable(pVCpu);
14781	HM_DISABLE_PREEMPT(pVCpu);
14782
14783	pCtx->dr[6] &= ~X86_DR6_B_MASK;
14784	pCtx->dr[6] \|= uDR6;
14785	if (CPUMIsGuestDebugStateActive(pVCpu))
14786	ASMSetDR6(pCtx->dr[6]);
14787
14788	HM_RESTORE_PREEMPT();
14789	VMMRZCallRing3Enable(pVCpu);
14790
14791	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
14792	AssertRCReturn(rc, rc);
14793
14794	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
14795	pCtx->dr[7] &= ~X86_DR7_GD;
14796
14797	/* Paranoia. */
14798	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
14799	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
14800
14801	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
14802	AssertRCReturn(rc, rc);
14803
14804	/*
14805	* Raise #DB in the guest.
14806	*
14807	* It is important to reflect exactly what the VM-exit gave us (preserving the
14808	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
14809	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
14810	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
14811	*
14812	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
14813	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
14814	*/
14815	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14816	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14817	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14818	AssertRCReturn(rc, rc);
14819	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14820	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14821	return VINF_SUCCESS;
14822	}
14823
14824	/*
14825	* Not a guest trap, must be a hypervisor related debug event then.
14826	* Update DR6 in case someone is interested in it.
14827	*/
14828	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
14829	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
14830	CPUMSetHyperDR6(pVCpu, uDR6);
14831
14832	return rc;
14833	}
14834
14835
14836	/**
14837	* Hacks its way around the lovely mesa driver's backdoor accesses.
14838	*
14839	* @sa hmR0SvmHandleMesaDrvGp
14840	*/
14841	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14842	{
14843	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
14844	RT_NOREF(pCtx);
14845
14846	/* For now we'll just skip the instruction. */
14847	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14848	}
14849
14850
14851	/**
14852	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
14853	* backdoor logging w/o checking what it is running inside.
14854	*
14855	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
14856	* backdoor port and magic numbers loaded in registers.
14857	*
14858	* @returns true if it is, false if it isn't.
14859	* @sa hmR0SvmIsMesaDrvGp
14860	*/
14861	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14862	{
14863	/* 0xed: IN eAX,dx */
14864	uint8_t abInstr[1];
14865	if (pVmxTransient->cbInstr != sizeof(abInstr))
14866	return false;
14867
14868	/* Check that it is #GP(0). */
14869	if (pVmxTransient->uExitIntErrorCode != 0)
14870	return false;
14871
14872	/* Check magic and port. */
14873	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
14874	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
14875	if (pCtx->rax != UINT32_C(0x564d5868))
14876	return false;
14877	if (pCtx->dx != UINT32_C(0x5658))
14878	return false;
14879
14880	/* Flat ring-3 CS. */
14881	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
14882	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
14883	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
14884	if (pCtx->cs.Attr.n.u2Dpl != 3)
14885	return false;
14886	if (pCtx->cs.u64Base != 0)
14887	return false;
14888
14889	/* Check opcode. */
14890	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
14891	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
14892	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
14893	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
14894	if (RT_FAILURE(rc))
14895	return false;
14896	if (abInstr[0] != 0xed)
14897	return false;
14898
14899	return true;
14900	}
14901
14902
14903	/**
14904	* VM-exit exception handler for \#GP (General-protection exception).
14905	*
14906	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
14907	*/
14908	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14909	{
14910	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14911	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
14912
14913	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14914	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14915	if (pVmcsInfo->RealMode.fRealOnV86Active)
14916	{ /* likely */ }
14917	else
14918	{
14919	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14920	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv);
14921	#endif
14922	/* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
14923	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14924	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14925	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14926	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14927	AssertRCReturn(rc, rc);
14928	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
14929	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
14930
14931	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
14932	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
14933	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14934	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14935	else
14936	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
14937	return rc;
14938	}
14939
14940	Assert(CPUMIsGuestInRealModeEx(pCtx));
14941	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
14942
14943	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14944	AssertRCReturn(rc, rc);
14945
14946	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
14947	if (rcStrict == VINF_SUCCESS)
14948	{
14949	if (!CPUMIsGuestInRealModeEx(pCtx))
14950	{
14951	/*
14952	* The guest is no longer in real-mode, check if we can continue executing the
14953	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
14954	*/
14955	pVmcsInfo->RealMode.fRealOnV86Active = false;
14956	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
14957	{
14958	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
14959	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14960	}
14961	else
14962	{
14963	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
14964	rcStrict = VINF_EM_RESCHEDULE;
14965	}
14966	}
14967	else
14968	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14969	}
14970	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14971	{
14972	rcStrict = VINF_SUCCESS;
14973	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14974	}
14975	return VBOXSTRICTRC_VAL(rcStrict);
14976	}
14977
14978
14979	/**
14980	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
14981	* the exception reported in the VMX transient structure back into the VM.
14982	*
14983	* @remarks Requires uExitIntInfo in the VMX transient structure to be
14984	* up-to-date.
14985	*/
14986	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14987	{
14988	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14989	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14990	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14991	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active,
14992	("uVector=%#x u32XcptBitmap=%#X32\n",
14993	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
14994	NOREF(pVmcsInfo);
14995	#endif
14996
14997	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
14998	hmR0VmxCheckExitDueToEventDelivery(). */
14999	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15000	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15001	AssertRCReturn(rc, rc);
15002	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15003
15004	#ifdef DEBUG_ramshankar
15005	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
15006	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n",
15007	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pCtx->cs.Sel, pCtx->rip));
15008	#endif
15009
15010	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15011	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15012	return VINF_SUCCESS;
15013	}
15014
15015
15016	/**
15017	* VM-exit exception handler for \#PF (Page-fault exception).
15018	*/
15019	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15020	{
15021	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15022	PVM pVM = pVCpu->CTX_SUFF(pVM);
15023	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15024	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15025	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15026	AssertRCReturn(rc, rc);
15027
15028	if (!pVM->hm.s.fNestedPaging)
15029	{ /* likely */ }
15030	else
15031	{
15032	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
15033	Assert(pVCpu->hm.s.fUsingDebugLoop);
15034	#endif
15035	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
15036	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
15037	{
15038	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15039	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
15040	}
15041	else
15042	{
15043	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15044	hmR0VmxSetPendingXcptDF(pVCpu);
15045	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
15046	}
15047	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15048	return rc;
15049	}
15050
15051	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
15052	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
15053	if (pVmxTransient->fVectoringPF)
15054	{
15055	Assert(pVCpu->hm.s.Event.fPending);
15056	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15057	}
15058
15059	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15060	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15061	AssertRCReturn(rc, rc);
15062
15063	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
15064	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
15065
15066	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
15067	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
15068
15069	Log4Func(("#PF: rc=%Rrc\n", rc));
15070	if (rc == VINF_SUCCESS)
15071	{
15072	/*
15073	* This is typically a shadow page table sync or a MMIO instruction. But we may have
15074	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
15075	*/
15076	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15077	TRPMResetTrap(pVCpu);
15078	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
15079	return rc;
15080	}
15081
15082	if (rc == VINF_EM_RAW_GUEST_TRAP)
15083	{
15084	if (!pVmxTransient->fVectoringDoublePF)
15085	{
15086	/* It's a guest page fault and needs to be reflected to the guest. */
15087	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
15088	TRPMResetTrap(pVCpu);
15089	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
15090	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15091	uGstErrorCode, pVmxTransient->uExitQual);
15092	}
15093	else
15094	{
15095	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15096	TRPMResetTrap(pVCpu);
15097	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
15098	hmR0VmxSetPendingXcptDF(pVCpu);
15099	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
15100	}
15101
15102	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15103	return VINF_SUCCESS;
15104	}
15105
15106	TRPMResetTrap(pVCpu);
15107	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
15108	return rc;
15109	}
15110
15111	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15112	/** @name VMX instruction handlers.
15113	* @{
15114	*/
15115	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15116	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VMX instructions VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15117	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15118
15119	/**
15120	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
15121	*/
15122	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15123	{
15124	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15125
15126	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15127	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15128	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15129	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15130	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15131	AssertRCReturn(rc, rc);
15132
15133	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15134
15135	VMXVEXITINFO ExitInfo;
15136	RT_ZERO(ExitInfo);
15137	ExitInfo.uReason = pVmxTransient->uExitReason;
15138	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15139	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15140	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15141	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15142
15143	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
15144	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15145	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15146	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15147	{
15148	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15149	rcStrict = VINF_SUCCESS;
15150	}
15151	return rcStrict;
15152	}
15153
15154
15155	/**
15156	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
15157	*/
15158	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15159	{
15160	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15161
15162	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
15163	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15164	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15165	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15166	AssertRCReturn(rc, rc);
15167
15168	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15169
15170	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
15171	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15172	{
15173	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15174	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15175	}
15176	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15177	return rcStrict;
15178	}
15179
15180
15181	/**
15182	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
15183	*/
15184	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15185	{
15186	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15187
15188	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15189	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15190	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15191	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15192	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15193	AssertRCReturn(rc, rc);
15194
15195	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15196
15197	VMXVEXITINFO ExitInfo;
15198	RT_ZERO(ExitInfo);
15199	ExitInfo.uReason = pVmxTransient->uExitReason;
15200	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15201	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15202	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15203	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15204
15205	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
15206	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15207	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15208	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15209	{
15210	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15211	rcStrict = VINF_SUCCESS;
15212	}
15213	return rcStrict;
15214	}
15215
15216
15217	/**
15218	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
15219	*/
15220	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15221	{
15222	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15223
15224	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15225	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15226	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15227	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15228	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15229	AssertRCReturn(rc, rc);
15230
15231	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15232
15233	VMXVEXITINFO ExitInfo;
15234	RT_ZERO(ExitInfo);
15235	ExitInfo.uReason = pVmxTransient->uExitReason;
15236	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15237	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15238	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15239	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15240
15241	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
15242	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15243	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15244	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15245	{
15246	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15247	rcStrict = VINF_SUCCESS;
15248	}
15249	return rcStrict;
15250	}
15251
15252
15253	/**
15254	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Unconditional VM-exit.
15255	*/
15256	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15257	{
15258	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15259
15260	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15261	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15262	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15263	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15264	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15265	AssertRCReturn(rc, rc);
15266
15267	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15268
15269	VMXVEXITINFO ExitInfo;
15270	RT_ZERO(ExitInfo);
15271	ExitInfo.uReason = pVmxTransient->uExitReason;
15272	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15273	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15274	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15275	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15276	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15277
15278	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
15279	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15280	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15281	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15282	{
15283	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15284	rcStrict = VINF_SUCCESS;
15285	}
15286	return rcStrict;
15287	}
15288
15289
15290	/**
15291	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
15292	*/
15293	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15294	{
15295	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15296
15297	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
15298	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15299	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15300	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15301	AssertRCReturn(rc, rc);
15302
15303	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15304
15305	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
15306	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15307	{
15308	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15309	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15310	}
15311	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15312	return rcStrict;
15313	}
15314
15315
15316	/**
15317	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Unconditional VM-exit.
15318	*/
15319	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15320	{
15321	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15322
15323	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15324	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15325	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15326	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15327	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15328	AssertRCReturn(rc, rc);
15329
15330	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15331
15332	VMXVEXITINFO ExitInfo;
15333	RT_ZERO(ExitInfo);
15334	ExitInfo.uReason = pVmxTransient->uExitReason;
15335	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15336	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15337	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15338	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15339	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15340
15341	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
15342	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15343	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15344	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15345	{
15346	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15347	rcStrict = VINF_SUCCESS;
15348	}
15349	return rcStrict;
15350	}
15351
15352
15353	/**
15354	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
15355	*/
15356	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15357	{
15358	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15359
15360	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15361	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
15362	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15363	AssertRCReturn(rc, rc);
15364
15365	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15366
15367	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
15368	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15369	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
15370	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15371	{
15372	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15373	rcStrict = VINF_SUCCESS;
15374	}
15375	return rcStrict;
15376	}
15377
15378
15379	/**
15380	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
15381	*/
15382	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15383	{
15384	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15385
15386	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15387	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15388	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15389	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15390	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15391	AssertRCReturn(rc, rc);
15392
15393	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15394
15395	VMXVEXITINFO ExitInfo;
15396	RT_ZERO(ExitInfo);
15397	ExitInfo.uReason = pVmxTransient->uExitReason;
15398	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15399	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15400	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15401	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15402
15403	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
15404	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15405	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15406	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15407	{
15408	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15409	rcStrict = VINF_SUCCESS;
15410	}
15411	return rcStrict;
15412	}
15413
15414	/** @} */
15415	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
15416

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 78259

Download in other formats: