HMVMXR0.cpp@ 78928

Last change on this file since 78928 was 78928, checked in by vboxsync, 6 years ago
VMM/HMVMXR0: Nested VMX: bugref:9180 Partial exception/NMI VM-exit handling.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 687.5 KB

Line
1	/* $Id: HMVMXR0.cpp 78928 2019-06-03 10:42:40Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27
28	#include <VBox/vmm/pdmapi.h>
29	#include <VBox/vmm/dbgf.h>
30	#include <VBox/vmm/iem.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#include <VBox/vmm/em.h>
35	#include <VBox/vmm/gim.h>
36	#include <VBox/vmm/apic.h>
37	#ifdef VBOX_WITH_REM
38	# include <VBox/vmm/rem.h>
39	#endif
40	#include "HMInternal.h"
41	#include <VBox/vmm/vm.h>
42	#include <VBox/vmm/hmvmxinline.h>
43	#include "HMVMXR0.h"
44	#include "dtrace/VBoxVMM.h"
45
46	#ifdef DEBUG_ramshankar
47	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
48	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
49	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
50	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
51	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
52	# define HMVMX_ALWAYS_TRAP_PF
53	# define HMVMX_ALWAYS_FLUSH_TLB
54	# define HMVMX_ALWAYS_SWAP_EFER
55	#endif
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	/** Use the function table. */
62	#define HMVMX_USE_FUNCTION_TABLE
63
64	/** Determine which tagged-TLB flush handler to use. */
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70	/** @name HMVMX_READ_XXX
71	* Flags to skip redundant reads of some common VMCS fields that are not part of
72	* the guest-CPU or VCPU state but are needed while handling VM-exits.
73	*/
74	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
75	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
76	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
77	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
78	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
79	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
80	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
81	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
82	/** @} */
83
84	/**
85	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
86	* guest using hardware-assisted VMX.
87	*
88	* This excludes state like GPRs (other than RSP) which are always are
89	* swapped and restored across the world-switch and also registers like EFER,
90	* MSR which cannot be modified by the guest without causing a VM-exit.
91	*/
92	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
93	\| CPUMCTX_EXTRN_RFLAGS \
94	\| CPUMCTX_EXTRN_RSP \
95	\| CPUMCTX_EXTRN_SREG_MASK \
96	\| CPUMCTX_EXTRN_TABLE_MASK \
97	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
98	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
99	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
100	\| CPUMCTX_EXTRN_TSC_AUX \
101	\| CPUMCTX_EXTRN_OTHER_MSRS \
102	\| CPUMCTX_EXTRN_CR0 \
103	\| CPUMCTX_EXTRN_CR3 \
104	\| CPUMCTX_EXTRN_CR4 \
105	\| CPUMCTX_EXTRN_DR7 \
106	\| CPUMCTX_EXTRN_HM_VMX_MASK)
107
108	/**
109	* Exception bitmap mask for real-mode guests (real-on-v86).
110	*
111	* We need to intercept all exceptions manually except:
112	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
113	* due to bugs in Intel CPUs.
114	* - \#PF need not be intercepted even in real-mode if we have nested paging
115	* support.
116	*/
117	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
118	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
119	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
120	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
121	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
122	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
123	\| RT_BIT(X86_XCPT_XF))
124
125	/** Maximum VM-instruction error number. */
126	#define HMVMX_INSTR_ERROR_MAX 28
127
128	/** Profiling macro. */
129	#ifdef HM_PROFILE_EXIT_DISPATCH
130	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
131	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
132	#else
133	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
134	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
135	#endif
136
137	/** Assert that preemption is disabled or covered by thread-context hooks. */
138	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
139	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
140
141	/** Assert that we haven't migrated CPUs when thread-context hooks are not
142	* used. */
143	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
144	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
145	("Illegal migration! Entered on CPU %u Current %u\n", \
146	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
147
148	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
149	* context. */
150	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
151	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
152	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
153
154	/** Helper macro for VM-exit handlers called unexpectedly. */
155	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
156	do { \
157	(a_pVCpu)->hm.s.u32HMError = (a_HmError); \
158	return VERR_VMX_UNEXPECTED_EXIT; \
159	} while (0)
160
161	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
162	/** Macro that does the necessary privilege checks and intercepted VM-exits for
163	* guests that attempted to execute a VMX instruction. */
164	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
165	do \
166	{ \
167	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
168	if (rcStrictTmp == VINF_SUCCESS) \
169	{ /* likely */ } \
170	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
171	{ \
172	Assert((a_pVCpu)->hm.s.Event.fPending); \
173	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
174	return VINF_SUCCESS; \
175	} \
176	else \
177	{ \
178	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
179	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
180	} \
181	} while (0)
182
183	/** Macro that decodes a memory operand for an instruction VM-exit. */
184	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
185	do \
186	{ \
187	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
188	(a_pGCPtrEffAddr)); \
189	if (rcStrictTmp == VINF_SUCCESS) \
190	{ /* likely */ } \
191	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
192	{ \
193	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
194	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
195	NOREF(uXcptTmp); \
196	return VINF_SUCCESS; \
197	} \
198	else \
199	{ \
200	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
201	return rcStrictTmp; \
202	} \
203	} while (0)
204
205	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
206
207
208	/*********************************************************************************************************************************
209	* Structures and Typedefs *
210	*********************************************************************************************************************************/
211	/**
212	* VMX transient state.
213	*
214	* A state structure for holding miscellaneous information across
215	* VMX non-root operation and restored after the transition.
216	*/
217	typedef struct VMXTRANSIENT
218	{
219	/** The host's rflags/eflags. */
220	RTCCUINTREG fEFlags;
221	#if HC_ARCH_BITS == 32
222	uint32_t u32Alignment0;
223	#endif
224	/** The guest's TPR value used for TPR shadowing. */
225	uint8_t u8GuestTpr;
226	/** Alignment. */
227	uint8_t abAlignment0[7];
228
229	/** The basic VM-exit reason. */
230	uint16_t uExitReason;
231	/** Alignment. */
232	uint16_t u16Alignment0;
233	/** The VM-exit interruption error code. */
234	uint32_t uExitIntErrorCode;
235	/** The VM-exit exit code qualification. */
236	uint64_t uExitQual;
237	/** The Guest-linear address. */
238	uint64_t uGuestLinearAddr;
239
240	/** The VM-exit interruption-information field. */
241	uint32_t uExitIntInfo;
242	/** The VM-exit instruction-length field. */
243	uint32_t cbInstr;
244	/** The VM-exit instruction-information field. */
245	VMXEXITINSTRINFO ExitInstrInfo;
246	/** Whether the VM-entry failed or not. */
247	bool fVMEntryFailed;
248	/** Whether we are currently executing a nested-guest. */
249	bool fIsNestedGuest;
250	/** Alignment. */
251	uint8_t abAlignment1[2];
252
253	/** The VM-entry interruption-information field. */
254	uint32_t uEntryIntInfo;
255	/** The VM-entry exception error code field. */
256	uint32_t uEntryXcptErrorCode;
257	/** The VM-entry instruction length field. */
258	uint32_t cbEntryInstr;
259
260	/** IDT-vectoring information field. */
261	uint32_t uIdtVectoringInfo;
262	/** IDT-vectoring error code. */
263	uint32_t uIdtVectoringErrorCode;
264
265	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
266	uint32_t fVmcsFieldsRead;
267
268	/** Whether the guest debug state was active at the time of VM-exit. */
269	bool fWasGuestDebugStateActive;
270	/** Whether the hyper debug state was active at the time of VM-exit. */
271	bool fWasHyperDebugStateActive;
272	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
273	bool fUpdatedTscOffsettingAndPreemptTimer;
274	/** Whether the VM-exit was caused by a page-fault during delivery of a
275	* contributory exception or a page-fault. */
276	bool fVectoringDoublePF;
277	/** Whether the VM-exit was caused by a page-fault during delivery of an
278	* external interrupt or NMI. */
279	bool fVectoringPF;
280	bool afAlignment0[3];
281
282	/** The VMCS info. object. */
283	PVMXVMCSINFO pVmcsInfo;
284	} VMXTRANSIENT;
285	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
286	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
287	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
288	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
289	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
290	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
291	/** Pointer to VMX transient state. */
292	typedef VMXTRANSIENT *PVMXTRANSIENT;
293
294	/**
295	* Memory operand read or write access.
296	*/
297	typedef enum VMXMEMACCESS
298	{
299	VMXMEMACCESS_READ = 0,
300	VMXMEMACCESS_WRITE = 1
301	} VMXMEMACCESS;
302
303	/**
304	* VMX VM-exit handler.
305	*
306	* @returns Strict VBox status code (i.e. informational status codes too).
307	* @param pVCpu The cross context virtual CPU structure.
308	* @param pVmxTransient The VMX-transient structure.
309	*/
310	#ifndef HMVMX_USE_FUNCTION_TABLE
311	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
312	#else
313	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
314	/** Pointer to VM-exit handler. */
315	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
316	#endif
317
318	/**
319	* VMX VM-exit handler, non-strict status code.
320	*
321	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
322	*
323	* @returns VBox status code, no informational status code returned.
324	* @param pVCpu The cross context virtual CPU structure.
325	* @param pVmxTransient The VMX-transient structure.
326	*
327	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
328	* use of that status code will be replaced with VINF_EM_SOMETHING
329	* later when switching over to IEM.
330	*/
331	#ifndef HMVMX_USE_FUNCTION_TABLE
332	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
333	#else
334	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
335	#endif
336
337
338	/*********************************************************************************************************************************
339	* Internal Functions *
340	*********************************************************************************************************************************/
341	#ifndef HMVMX_USE_FUNCTION_TABLE
342	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
343	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
344	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
345	#else
346	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
347	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
348	#endif
349	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
350	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
351	#endif
352
353	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
354	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
355	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
356	#endif
357
358	/** @name VM-exit handlers.
359	* @{
360	*/
361	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
362	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
363	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
364	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
365	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
366	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
367	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
368	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
369	static FNVMXEXITHANDLER hmR0VmxExitHlt;
370	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
371	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
372	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
373	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
374	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
375	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
376	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
377	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
378	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
379	static FNVMXEXITHANDLER hmR0VmxExitVmread;
380	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
381	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
382	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
383	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
384	static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
385	#endif
386	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
387	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
388	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
389	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
390	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
391	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
392	static FNVMXEXITHANDLER hmR0VmxExitMwait;
393	static FNVMXEXITHANDLER hmR0VmxExitMtf;
394	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
395	static FNVMXEXITHANDLER hmR0VmxExitPause;
396	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
397	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
398	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
399	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
400	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
401	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
402	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
403	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
404	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
405	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
406	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
407	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
408	/** @} */
409
410	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
411	/** @name Nested-guest VM-exit handlers.
412	* @{
413	*/
414	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmiNested;
415	//static FNVMXEXITHANDLER hmR0VmxExitExtIntNested;
416	static FNVMXEXITHANDLER hmR0VmxExitTripleFaultNested;
417	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindowNested;
418	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindowNested;
419	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitchNested;
420	//static FNVMXEXITHANDLER hmR0VmxExitCpuid;
421	//static FNVMXEXITHANDLER hmR0VmxExitGetsec;
422	static FNVMXEXITHANDLER hmR0VmxExitHltNested;
423	//static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
424	static FNVMXEXITHANDLER hmR0VmxExitInvlpgNested;
425	static FNVMXEXITHANDLER hmR0VmxExitRdpmcNested;
426	//static FNVMXEXITHANDLER hmR0VmxExitVmcall;
427	//static FNVMXEXITHANDLER hmR0VmxExitVmclear;
428	//static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
429	//static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
430	//static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
431	static FNVMXEXITHANDLER hmR0VmxExitVmreadVmwriteNested;
432	//static FNVMXEXITHANDLER hmR0VmxExitVmresume;
433	//static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
434	//static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
435	//static FNVMXEXITHANDLER hmR0VmxExitVmxon;
436	//static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
437	static FNVMXEXITHANDLER hmR0VmxExitRdtscNested;
438	static FNVMXEXITHANDLER hmR0VmxExitMovCRxNested;
439	static FNVMXEXITHANDLER hmR0VmxExitMovDRxNested;
440	static FNVMXEXITHANDLER hmR0VmxExitIoInstrNested;
441	static FNVMXEXITHANDLER hmR0VmxExitRdmsrNested;
442	static FNVMXEXITHANDLER hmR0VmxExitWrmsrNested;
443	static FNVMXEXITHANDLER hmR0VmxExitMwaitNested;
444	static FNVMXEXITHANDLER hmR0VmxExitMtfNested;
445	static FNVMXEXITHANDLER hmR0VmxExitMonitorNested;
446	static FNVMXEXITHANDLER hmR0VmxExitPauseNested;
447	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThresholdNested;
448	static FNVMXEXITHANDLER hmR0VmxExitApicAccessNested;
449	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccessNested;
450	//static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
451	//static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
452	static FNVMXEXITHANDLER hmR0VmxExitRdtscpNested;
453	//static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
454	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvdNested;
455	//static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
456	//static FNVMXEXITHANDLER hmR0VmxExitErrUnexpected;
457	static FNVMXEXITHANDLER hmR0VmxExitInvpcidNested;
458	//static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
459	//static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
460	//static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
461	static FNVMXEXITHANDLER hmR0VmxExitInstrNested;
462	static FNVMXEXITHANDLER hmR0VmxExitInstrWithInfoNested;
463	/** @} */
464	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
465
466	/** @name Helpers for hardware exceptions VM-exit handlers.
467	* @{
468	*/
469	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
470	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
471	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
472	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
473	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
474	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
475	static VBOXSTRICTRC hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
476	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst);
477	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr);
478	static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg);
479	static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg);
480	static VBOXSTRICTRC hmR0VmxExitHostNmi(PVMCPU pVCpu);
481	/** @} */
482
483
484	/*********************************************************************************************************************************
485	* Global Variables *
486	*********************************************************************************************************************************/
487	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
488	static const uint32_t g_aVmcsCacheSegBase[] =
489	{
490	VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
491	VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
492	VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
493	VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
494	VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
495	VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
496	};
497	AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
498	#endif
499	static const uint32_t g_aVmcsSegBase[] =
500	{
501	VMX_VMCS_GUEST_ES_BASE,
502	VMX_VMCS_GUEST_CS_BASE,
503	VMX_VMCS_GUEST_SS_BASE,
504	VMX_VMCS_GUEST_DS_BASE,
505	VMX_VMCS_GUEST_FS_BASE,
506	VMX_VMCS_GUEST_GS_BASE
507	};
508	static const uint32_t g_aVmcsSegSel[] =
509	{
510	VMX_VMCS16_GUEST_ES_SEL,
511	VMX_VMCS16_GUEST_CS_SEL,
512	VMX_VMCS16_GUEST_SS_SEL,
513	VMX_VMCS16_GUEST_DS_SEL,
514	VMX_VMCS16_GUEST_FS_SEL,
515	VMX_VMCS16_GUEST_GS_SEL
516	};
517	static const uint32_t g_aVmcsSegLimit[] =
518	{
519	VMX_VMCS32_GUEST_ES_LIMIT,
520	VMX_VMCS32_GUEST_CS_LIMIT,
521	VMX_VMCS32_GUEST_SS_LIMIT,
522	VMX_VMCS32_GUEST_DS_LIMIT,
523	VMX_VMCS32_GUEST_FS_LIMIT,
524	VMX_VMCS32_GUEST_GS_LIMIT
525	};
526	static const uint32_t g_aVmcsSegAttr[] =
527	{
528	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
529	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
530	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
531	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
532	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
533	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
534	};
535	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
536	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
537	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
538	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
539
540	#ifdef HMVMX_USE_FUNCTION_TABLE
541	/**
542	* VMX_EXIT dispatch table.
543	*/
544	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
545	{
546	/* 0 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
547	/* 1 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
548	/* 2 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
549	/* 3 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitErrUnexpected,
550	/* 4 VMX_EXIT_SIPI */ hmR0VmxExitErrUnexpected,
551	/* 5 VMX_EXIT_IO_SMI */ hmR0VmxExitErrUnexpected,
552	/* 6 VMX_EXIT_SMI */ hmR0VmxExitErrUnexpected,
553	/* 7 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
554	/* 8 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
555	/* 9 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
556	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
557	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
558	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
559	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
560	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
561	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
562	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
563	/* 17 VMX_EXIT_RSM */ hmR0VmxExitErrUnexpected,
564	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
565	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
566	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
567	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
568	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
569	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
570	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
571	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
572	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
573	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
574	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
575	#else
576	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
577	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
578	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
579	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
580	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
581	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
582	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
583	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
584	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
585	#endif
586	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
587	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
588	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
589	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
590	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
591	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
592	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrUnexpected,
593	/* 35 UNDEFINED */ hmR0VmxExitErrUnexpected,
594	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
595	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
596	/* 38 UNDEFINED */ hmR0VmxExitErrUnexpected,
597	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
598	/* 40 VMX_EXIT_PAUSE */ hmR0VmxExitPause,
599	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUnexpected,
600	/* 42 UNDEFINED */ hmR0VmxExitErrUnexpected,
601	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
602	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
603	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ hmR0VmxExitErrUnexpected,
604	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitErrUnexpected,
605	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitErrUnexpected,
606	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
607	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
608	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
609	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
610	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
611	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
612	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInvvpid,
613	#else
614	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
615	#endif
616	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
617	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
618	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUnexpected,
619	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitErrUnexpected,
620	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
621	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitErrUnexpected,
622	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUnexpected,
623	/* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUnexpected,
624	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUnexpected,
625	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitErrUnexpected,
626	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitErrUnexpected,
627	/* 65 UNDEFINED */ hmR0VmxExitErrUnexpected,
628	/* 66 VMX_EXIT_SPP_EVENT */ hmR0VmxExitErrUnexpected,
629	/* 67 VMX_EXIT_UMWAIT */ hmR0VmxExitErrUnexpected,
630	/* 68 VMX_EXIT_TPAUSE */ hmR0VmxExitErrUnexpected,
631	};
632	#endif /* HMVMX_USE_FUNCTION_TABLE */
633
634	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
635	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
636	{
637	/* 0 */ "(Not Used)",
638	/* 1 */ "VMCALL executed in VMX root operation.",
639	/* 2 */ "VMCLEAR with invalid physical address.",
640	/* 3 */ "VMCLEAR with VMXON pointer.",
641	/* 4 */ "VMLAUNCH with non-clear VMCS.",
642	/* 5 */ "VMRESUME with non-launched VMCS.",
643	/* 6 */ "VMRESUME after VMXOFF",
644	/* 7 */ "VM-entry with invalid control fields.",
645	/* 8 */ "VM-entry with invalid host state fields.",
646	/* 9 */ "VMPTRLD with invalid physical address.",
647	/* 10 */ "VMPTRLD with VMXON pointer.",
648	/* 11 */ "VMPTRLD with incorrect revision identifier.",
649	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
650	/* 13 */ "VMWRITE to read-only VMCS component.",
651	/* 14 */ "(Not Used)",
652	/* 15 */ "VMXON executed in VMX root operation.",
653	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
654	/* 17 */ "VM-entry with non-launched executing VMCS.",
655	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
656	/* 19 */ "VMCALL with non-clear VMCS.",
657	/* 20 */ "VMCALL with invalid VM-exit control fields.",
658	/* 21 */ "(Not Used)",
659	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
660	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
661	/* 24 */ "VMCALL with invalid SMM-monitor features.",
662	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
663	/* 26 */ "VM-entry with events blocked by MOV SS.",
664	/* 27 */ "(Not Used)",
665	/* 28 */ "Invalid operand to INVEPT/INVVPID."
666	};
667	#endif /* VBOX_STRICT */
668
669
670	/**
671	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
672	*
673	* Any bit set in this mask is owned by the host/hypervisor and would cause a
674	* VM-exit when modified by the guest.
675	*
676	* @returns The static CR0 guest/host mask.
677	* @param pVCpu The cross context virtual CPU structure.
678	*/
679	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PCVMCPU pVCpu)
680	{
681	/*
682	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
683	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
684	*/
685	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
686	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
687	* and @bugref{6944}. */
688	PVM pVM = pVCpu->CTX_SUFF(pVM);
689	return ( X86_CR0_PE
690	\| X86_CR0_NE
691	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
692	\| X86_CR0_PG
693	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
694	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
695	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
696	}
697
698
699	/**
700	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
701	*
702	* Any bit set in this mask is owned by the host/hypervisor and would cause a
703	* VM-exit when modified by the guest.
704	*
705	* @returns The static CR4 guest/host mask.
706	* @param pVCpu The cross context virtual CPU structure.
707	*/
708	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PCVMCPU pVCpu)
709	{
710	/*
711	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
712	* these bits are reserved on hardware that does not support them. Since the
713	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
714	* these bits and handle it depending on whether we expose them to the guest.
715	*/
716	PVM pVM = pVCpu->CTX_SUFF(pVM);
717	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
718	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
719	return ( X86_CR4_VMXE
720	\| X86_CR4_VME
721	\| X86_CR4_PAE
722	\| X86_CR4_PGE
723	\| X86_CR4_PSE
724	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
725	\| (fPcid ? X86_CR4_PCIDE : 0));
726	}
727
728
729	/**
730	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
731	* area.
732	*
733	* @returns @c true if it's different, @c false otherwise.
734	* @param pVmcsInfo The VMCS info. object.
735	*/
736	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
737	{
738	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
739	&& pVmcsInfo->pvGuestMsrStore);
740	}
741
742
743	/**
744	* Adds one or more exceptions to the exception bitmap and commits it to the current
745	* VMCS.
746	*
747	* @returns VBox status code.
748	* @param pVmxTransient The VMX-transient structure.
749	* @param uXcptMask The exception(s) to add.
750	*/
751	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
752	{
753	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
754	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
755	if ((uXcptBitmap & uXcptMask) != uXcptMask)
756	{
757	uXcptBitmap \|= uXcptMask;
758	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
759	AssertRCReturn(rc, rc);
760	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
761	}
762	return VINF_SUCCESS;
763	}
764
765
766	/**
767	* Adds an exception to the exception bitmap and commits it to the current VMCS.
768	*
769	* @returns VBox status code.
770	* @param pVmxTransient The VMX-transient structure.
771	* @param uXcpt The exception to add.
772	*/
773	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
774	{
775	Assert(uXcpt <= X86_XCPT_LAST);
776	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
777	}
778
779
780	/**
781	* Remove one or more exceptions from the exception bitmap and commits it to the
782	* current VMCS.
783	*
784	* This takes care of not removing the exception intercept if a nested-guest
785	* requires the exception to be intercepted.
786	*
787	* @returns VBox status code.
788	* @param pVCpu The cross context virtual CPU structure.
789	* @param pVmxTransient The VMX-transient structure.
790	* @param uXcptMask The exception(s) to remove.
791	*/
792	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
793	{
794	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
795	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
796	if (u32XcptBitmap & uXcptMask)
797	{
798	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
799	if (!pVmxTransient->fIsNestedGuest)
800	{ /* likely */ }
801	else
802	{
803	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
804	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
805	}
806	#endif
807	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
808	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
809	\| RT_BIT(X86_XCPT_DE)
810	\| RT_BIT(X86_XCPT_NM)
811	\| RT_BIT(X86_XCPT_TS)
812	\| RT_BIT(X86_XCPT_UD)
813	\| RT_BIT(X86_XCPT_NP)
814	\| RT_BIT(X86_XCPT_SS)
815	\| RT_BIT(X86_XCPT_GP)
816	\| RT_BIT(X86_XCPT_PF)
817	\| RT_BIT(X86_XCPT_MF));
818	#elif defined(HMVMX_ALWAYS_TRAP_PF)
819	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
820	#endif
821	if (uXcptMask)
822	{
823	/* Validate we are not removing any essential exception intercepts. */
824	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
825	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
826	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
827
828	/* Remove it from the exception bitmap. */
829	u32XcptBitmap &= ~uXcptMask;
830
831	/* Commit and update the cache if necessary. */
832	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
833	{
834	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
835	AssertRCReturn(rc, rc);
836	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
837	}
838	}
839	}
840	return VINF_SUCCESS;
841	}
842
843
844	/**
845	* Remove an exceptions from the exception bitmap and commits it to the current
846	* VMCS.
847	*
848	* @returns VBox status code.
849	* @param pVCpu The cross context virtual CPU structure.
850	* @param pVmxTransient The VMX-transient structure.
851	* @param uXcpt The exception to remove.
852	*/
853	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
854	{
855	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
856	}
857
858
859	/**
860	* Loads the VMCS specified by the VMCS info. object.
861	*
862	* @returns VBox status code.
863	* @param pVmcsInfo The VMCS info. object.
864	*/
865	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
866	{
867	Assert(pVmcsInfo);
868	Assert(pVmcsInfo->HCPhysVmcs);
869	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
870
871	if (pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_CLEAR)
872	{
873	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
874	if (RT_SUCCESS(rc))
875	{
876	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
877	return VINF_SUCCESS;
878	}
879	return rc;
880	}
881	return VERR_VMX_INVALID_VMCS_LAUNCH_STATE;
882	}
883
884
885	/**
886	* Clears the VMCS specified by the VMCS info. object.
887	*
888	* @returns VBox status code.
889	* @param pVmcsInfo The VMCS info. object.
890	*/
891	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
892	{
893	Assert(pVmcsInfo);
894	Assert(pVmcsInfo->HCPhysVmcs);
895	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
896
897	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
898	if (RT_SUCCESS(rc))
899	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
900	return rc;
901	}
902
903
904	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
905	/**
906	* Switches the current VMCS to the one specified.
907	*
908	* @returns VBox status code.
909	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
910	* @param pVmcsInfoTo The VMCS info. object we are switching to.
911	*
912	* @remarks Called with interrupts disabled.
913	*/
914	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
915	{
916	Assert(pVmcsInfoFrom);
917	Assert(pVmcsInfoTo);
918
919	/*
920	* Clear the VMCS we are switching out if it has not already been cleared.
921	* This will sync any CPU internal data back to the VMCS.
922	*/
923	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
924	{
925	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
926	if (RT_SUCCESS(rc))
927	{ /* likely */ }
928	else
929	return rc;
930	}
931
932	/*
933	* Clear the VMCS we are switching to if it has not already been cleared.
934	* This will initialize the VMCS launch state to "clear" required for loading it.
935	*
936	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
937	*/
938	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
939	{
940	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
941	if (RT_SUCCESS(rc))
942	{ /* likely */ }
943	else
944	return rc;
945	}
946
947	/*
948	* Finally, load the VMCS we are switching to.
949	*/
950	return hmR0VmxLoadVmcs(pVmcsInfoTo);
951	}
952	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
953
954
955	/**
956	* Updates the VM's last error record.
957	*
958	* If there was a VMX instruction error, reads the error data from the VMCS and
959	* updates VCPU's last error record as well.
960	*
961	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
962	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
963	* VERR_VMX_INVALID_VMCS_FIELD.
964	* @param rc The error code.
965	*/
966	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
967	{
968	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
969	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
970	{
971	AssertPtrReturnVoid(pVCpu);
972	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
973	}
974	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
975	}
976
977
978	#ifdef VBOX_STRICT
979	/**
980	* Reads the VM-entry interruption-information field from the VMCS into the VMX
981	* transient structure.
982	*
983	* @returns VBox status code.
984	* @param pVmxTransient The VMX-transient structure.
985	*
986	* @remarks No-long-jump zone!!!
987	*/
988	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
989	{
990	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
991	AssertRCReturn(rc, rc);
992	return VINF_SUCCESS;
993	}
994
995
996	/**
997	* Reads the VM-entry exception error code field from the VMCS into
998	* the VMX transient structure.
999	*
1000	* @returns VBox status code.
1001	* @param pVmxTransient The VMX-transient structure.
1002	*
1003	* @remarks No-long-jump zone!!!
1004	*/
1005	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1006	{
1007	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1008	AssertRCReturn(rc, rc);
1009	return VINF_SUCCESS;
1010	}
1011
1012
1013	/**
1014	* Reads the VM-entry exception error code field from the VMCS into
1015	* the VMX transient structure.
1016	*
1017	* @returns VBox status code.
1018	* @param pVmxTransient The VMX-transient structure.
1019	*
1020	* @remarks No-long-jump zone!!!
1021	*/
1022	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1023	{
1024	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1025	AssertRCReturn(rc, rc);
1026	return VINF_SUCCESS;
1027	}
1028	#endif /* VBOX_STRICT */
1029
1030
1031	/**
1032	* Reads the VM-exit interruption-information field from the VMCS into the VMX
1033	* transient structure.
1034	*
1035	* @returns VBox status code.
1036	* @param pVmxTransient The VMX-transient structure.
1037	*/
1038	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1039	{
1040	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1041	{
1042	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1043	AssertRCReturn(rc,rc);
1044	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
1045	}
1046	return VINF_SUCCESS;
1047	}
1048
1049
1050	/**
1051	* Reads the VM-exit interruption error code from the VMCS into the VMX
1052	* transient structure.
1053	*
1054	* @returns VBox status code.
1055	* @param pVmxTransient The VMX-transient structure.
1056	*/
1057	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1058	{
1059	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1060	{
1061	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1062	AssertRCReturn(rc, rc);
1063	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1064	}
1065	return VINF_SUCCESS;
1066	}
1067
1068
1069	/**
1070	* Reads the VM-exit instruction length field from the VMCS into the VMX
1071	* transient structure.
1072	*
1073	* @returns VBox status code.
1074	* @param pVmxTransient The VMX-transient structure.
1075	*/
1076	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1077	{
1078	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1079	{
1080	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1081	AssertRCReturn(rc, rc);
1082	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1083	}
1084	return VINF_SUCCESS;
1085	}
1086
1087
1088	/**
1089	* Reads the VM-exit instruction-information field from the VMCS into
1090	* the VMX transient structure.
1091	*
1092	* @returns VBox status code.
1093	* @param pVmxTransient The VMX-transient structure.
1094	*/
1095	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1096	{
1097	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1098	{
1099	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1100	AssertRCReturn(rc, rc);
1101	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1102	}
1103	return VINF_SUCCESS;
1104	}
1105
1106
1107	/**
1108	* Reads the VM-exit Qualification from the VMCS into the VMX transient structure.
1109	*
1110	* @returns VBox status code.
1111	* @param pVCpu The cross context virtual CPU structure of the
1112	* calling EMT. (Required for the VMCS cache case.)
1113	* @param pVmxTransient The VMX-transient structure.
1114	*/
1115	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1116	{
1117	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1118	{
1119	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1120	AssertRCReturn(rc, rc);
1121	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1122	}
1123	return VINF_SUCCESS;
1124	}
1125
1126
1127	/**
1128	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1129	*
1130	* @returns VBox status code.
1131	* @param pVCpu The cross context virtual CPU structure of the
1132	* calling EMT. (Required for the VMCS cache case.)
1133	* @param pVmxTransient The VMX-transient structure.
1134	*/
1135	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1136	{
1137	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1138	{
1139	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1140	AssertRCReturn(rc, rc);
1141	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1142	}
1143	return VINF_SUCCESS;
1144	}
1145
1146
1147	/**
1148	* Reads the IDT-vectoring information field from the VMCS into the VMX
1149	* transient structure.
1150	*
1151	* @returns VBox status code.
1152	* @param pVmxTransient The VMX-transient structure.
1153	*
1154	* @remarks No-long-jump zone!!!
1155	*/
1156	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1157	{
1158	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1159	{
1160	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1161	AssertRCReturn(rc, rc);
1162	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1163	}
1164	return VINF_SUCCESS;
1165	}
1166
1167
1168	/**
1169	* Reads the IDT-vectoring error code from the VMCS into the VMX
1170	* transient structure.
1171	*
1172	* @returns VBox status code.
1173	* @param pVmxTransient The VMX-transient structure.
1174	*/
1175	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1176	{
1177	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1178	{
1179	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1180	AssertRCReturn(rc, rc);
1181	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1182	}
1183	return VINF_SUCCESS;
1184	}
1185
1186
1187	/**
1188	* Enters VMX root mode operation on the current CPU.
1189	*
1190	* @returns VBox status code.
1191	* @param pVM The cross context VM structure. Can be
1192	* NULL, after a resume.
1193	* @param HCPhysCpuPage Physical address of the VMXON region.
1194	* @param pvCpuPage Pointer to the VMXON region.
1195	*/
1196	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1197	{
1198	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1199	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1200	Assert(pvCpuPage);
1201	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1202
1203	if (pVM)
1204	{
1205	/* Write the VMCS revision identifier to the VMXON region. */
1206	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1207	}
1208
1209	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1210	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1211
1212	/* Enable the VMX bit in CR4 if necessary. */
1213	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1214
1215	/* Enter VMX root mode. */
1216	int rc = VMXEnable(HCPhysCpuPage);
1217	if (RT_FAILURE(rc))
1218	{
1219	if (!(uOldCr4 & X86_CR4_VMXE))
1220	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1221
1222	if (pVM)
1223	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1224	}
1225
1226	/* Restore interrupts. */
1227	ASMSetFlags(fEFlags);
1228	return rc;
1229	}
1230
1231
1232	/**
1233	* Exits VMX root mode operation on the current CPU.
1234	*
1235	* @returns VBox status code.
1236	*/
1237	static int hmR0VmxLeaveRootMode(void)
1238	{
1239	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1240
1241	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1242	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1243
1244	/* If we're for some reason not in VMX root mode, then don't leave it. */
1245	RTCCUINTREG const uHostCR4 = ASMGetCR4();
1246
1247	int rc;
1248	if (uHostCR4 & X86_CR4_VMXE)
1249	{
1250	/* Exit VMX root mode and clear the VMX bit in CR4. */
1251	VMXDisable();
1252	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1253	rc = VINF_SUCCESS;
1254	}
1255	else
1256	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1257
1258	/* Restore interrupts. */
1259	ASMSetFlags(fEFlags);
1260	return rc;
1261	}
1262
1263
1264	/**
1265	* Allocates and maps a physically contiguous page. The allocated page is
1266	* zero'd out (used by various VT-x structures).
1267	*
1268	* @returns IPRT status code.
1269	* @param pMemObj Pointer to the ring-0 memory object.
1270	* @param ppVirt Where to store the virtual address of the
1271	* allocation.
1272	* @param pHCPhys Where to store the physical address of the
1273	* allocation.
1274	*/
1275	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1276	{
1277	AssertPtr(pMemObj);
1278	AssertPtr(ppVirt);
1279	AssertPtr(pHCPhys);
1280	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1281	if (RT_FAILURE(rc))
1282	return rc;
1283	ppVirt = RTR0MemObjAddress(pMemObj);
1284	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1285	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1286	return VINF_SUCCESS;
1287	}
1288
1289
1290	/**
1291	* Frees and unmaps an allocated, physical page.
1292	*
1293	* @param pMemObj Pointer to the ring-0 memory object.
1294	* @param ppVirt Where to re-initialize the virtual address of
1295	* allocation as 0.
1296	* @param pHCPhys Where to re-initialize the physical address of the
1297	* allocation as 0.
1298	*/
1299	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1300	{
1301	AssertPtr(pMemObj);
1302	AssertPtr(ppVirt);
1303	AssertPtr(pHCPhys);
1304	/* NULL is valid, accepted and ignored by the free function below. */
1305	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1306	*pMemObj = NIL_RTR0MEMOBJ;
1307	*ppVirt = NULL;
1308	*pHCPhys = NIL_RTHCPHYS;
1309	}
1310
1311
1312	/**
1313	* Initializes a VMCS info. object.
1314	*
1315	* @param pVmcsInfo The VMCS info. object.
1316	*/
1317	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1318	{
1319	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1320
1321	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1322	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1323	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1324	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1325	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1326	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1327	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1328	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1329	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1330	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1331	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1332	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1333	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1334	}
1335
1336
1337	/**
1338	* Frees the VT-x structures for a VMCS info. object.
1339	*
1340	* @param pVM The cross context VM structure.
1341	* @param pVmcsInfo The VMCS info. object.
1342	*/
1343	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1344	{
1345	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1346
1347	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1348	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1349
1350	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1351	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1352	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1353
1354	hmR0VmxInitVmcsInfo(pVmcsInfo);
1355	}
1356
1357
1358	/**
1359	* Allocates the VT-x structures for a VMCS info. object.
1360	*
1361	* @returns VBox status code.
1362	* @param pVCpu The cross context virtual CPU structure.
1363	* @param pVmcsInfo The VMCS info. object.
1364	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1365	*/
1366	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1367	{
1368	PVM pVM = pVCpu->CTX_SUFF(pVM);
1369
1370	/* Allocate the guest VM control structure (VMCS). */
1371	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1372	if (RT_SUCCESS(rc))
1373	{
1374	if (!fIsNstGstVmcs)
1375	{
1376	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1377	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1378	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1379	{
1380	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1381	NULL /* pR3Ptr /, NULL / pRCPtr */);
1382	}
1383	}
1384	else
1385	{
1386	Assert(pVmcsInfo->HCPhysVirtApic == NIL_RTHCPHYS);
1387	Assert(!pVmcsInfo->pbVirtApic);
1388	}
1389
1390	if (RT_SUCCESS(rc))
1391	{
1392	/*
1393	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1394	* transparent accesses of specific MSRs.
1395	*
1396	* If the condition for enabling MSR bitmaps changes here, don't forget to
1397	* update HMIsMsrBitmapActive().
1398	*
1399	* We don't share MSR bitmaps between the guest and nested-guest as we then
1400	* don't need to care about carefully restoring the guest MSR bitmap.
1401	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1402	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1403	*/
1404	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1405	{
1406	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1407	if (RT_SUCCESS(rc))
1408	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1409	}
1410
1411	if (RT_SUCCESS(rc))
1412	{
1413	/*
1414	* Allocate the VM-entry MSR-load area for the guest MSRs.
1415	*
1416	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1417	* the guest and nested-guest.
1418	*/
1419	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1420	&pVmcsInfo->HCPhysGuestMsrLoad);
1421	if (RT_SUCCESS(rc))
1422	{
1423	/*
1424	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1425	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1426	*/
1427	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1428	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1429	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1430
1431	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1432	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1433	&pVmcsInfo->HCPhysHostMsrLoad);
1434	}
1435	}
1436	}
1437	}
1438
1439	return rc;
1440	}
1441
1442
1443	/**
1444	* Free all VT-x structures for the VM.
1445	*
1446	* @returns IPRT status code.
1447	* @param pVM The cross context VM structure.
1448	*/
1449	static void hmR0VmxStructsFree(PVM pVM)
1450	{
1451	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1452	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1453	#endif
1454	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1455
1456	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1457	{
1458	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1459	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1460	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1461	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1462	if (pVM->cpum.ro.GuestFeatures.fVmx)
1463	{
1464	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1465	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1466	}
1467	#endif
1468	}
1469	}
1470
1471
1472	/**
1473	* Allocate all VT-x structures for the VM.
1474	*
1475	* @returns IPRT status code.
1476	* @param pVM The cross context VM structure.
1477	*/
1478	static int hmR0VmxStructsAlloc(PVM pVM)
1479	{
1480	/*
1481	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1482	* The VMCS size cannot be more than 4096 bytes.
1483	*
1484	* See Intel spec. Appendix A.1 "Basic VMX Information".
1485	*/
1486	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1487	if (cbVmcs <= X86_PAGE_4K_SIZE)
1488	{ /* likely */ }
1489	else
1490	{
1491	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1492	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1493	}
1494
1495	/*
1496	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1497	*/
1498	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1499	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1500	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1501	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1502	#endif
1503
1504	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1505	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1506	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1507
1508	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1509	{
1510	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1511	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1512	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1513	}
1514
1515	/*
1516	* Allocate per-VM VT-x structures.
1517	*/
1518	int rc = VINF_SUCCESS;
1519	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1520	/* Allocate crash-dump magic scratch page. */
1521	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1522	if (RT_FAILURE(rc))
1523	{
1524	hmR0VmxStructsFree(pVM);
1525	return rc;
1526	}
1527	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1528	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1529	#endif
1530
1531	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1532	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1533	{
1534	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1535	&pVM->hm.s.vmx.HCPhysApicAccess);
1536	if (RT_FAILURE(rc))
1537	{
1538	hmR0VmxStructsFree(pVM);
1539	return rc;
1540	}
1541	}
1542
1543	/*
1544	* Initialize per-VCPU VT-x structures.
1545	*/
1546	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1547	{
1548	/* Allocate the guest VMCS structures. */
1549	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1550	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
1551	if (RT_SUCCESS(rc))
1552	{
1553	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1554	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
1555	if (pVM->cpum.ro.GuestFeatures.fVmx)
1556	{
1557	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
1558	if (RT_SUCCESS(rc))
1559	{ /* likely */ }
1560	else
1561	break;
1562	}
1563	#endif
1564	}
1565	else
1566	break;
1567	}
1568
1569	if (RT_FAILURE(rc))
1570	{
1571	hmR0VmxStructsFree(pVM);
1572	return rc;
1573	}
1574
1575	return VINF_SUCCESS;
1576	}
1577
1578
1579	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1580	/**
1581	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
1582	*
1583	* @returns @c true if the MSR is intercepted, @c false otherwise.
1584	* @param pvMsrBitmap The MSR bitmap.
1585	* @param offMsr The MSR byte offset.
1586	* @param iBit The bit offset from the byte offset.
1587	*/
1588	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
1589	{
1590	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
1591	Assert(pbMsrBitmap);
1592	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
1593	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
1594	}
1595	#endif
1596
1597
1598	/**
1599	* Sets the permission bits for the specified MSR in the given MSR bitmap.
1600	*
1601	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
1602	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
1603	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
1604	* the read/write access of this MSR.
1605	*
1606	* @param pVCpu The cross context virtual CPU structure.
1607	* @param pVmcsInfo The VMCS info. object.
1608	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1609	* @param idMsr The MSR value.
1610	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
1611	* include both a read -and- a write permission!
1612	*
1613	* @sa CPUMGetVmxMsrPermission.
1614	* @remarks Can be called with interrupts disabled.
1615	*/
1616	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
1617	{
1618	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
1619	Assert(pbMsrBitmap);
1620	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
1621
1622	/*
1623	* MSR-bitmap Layout:
1624	* Byte index MSR range Interpreted as
1625	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1626	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1627	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1628	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1629	*
1630	* A bit corresponding to an MSR within the above range causes a VM-exit
1631	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
1632	* the MSR range, it always cause a VM-exit.
1633	*
1634	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
1635	*/
1636	uint16_t const offBitmapRead = 0;
1637	uint16_t const offBitmapWrite = 0x800;
1638	uint16_t offMsr;
1639	int32_t iBit;
1640	if (idMsr <= UINT32_C(0x00001fff))
1641	{
1642	offMsr = 0;
1643	iBit = idMsr;
1644	}
1645	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1646	{
1647	offMsr = 0x400;
1648	iBit = idMsr - UINT32_C(0xc0000000);
1649	}
1650	else
1651	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
1652
1653	/*
1654	* Set the MSR read permission.
1655	*/
1656	uint16_t const offMsrRead = offBitmapRead + offMsr;
1657	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
1658	if (fMsrpm & VMXMSRPM_ALLOW_RD)
1659	{
1660	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1661	bool const fClear = !fIsNstGstVmcs ? true
1662	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
1663	#else
1664	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1665	bool const fClear = true;
1666	#endif
1667	if (fClear)
1668	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
1669	}
1670	else
1671	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
1672
1673	/*
1674	* Set the MSR write permission.
1675	*/
1676	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
1677	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
1678	if (fMsrpm & VMXMSRPM_ALLOW_WR)
1679	{
1680	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1681	bool const fClear = !fIsNstGstVmcs ? true
1682	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
1683	#else
1684	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1685	bool const fClear = true;
1686	#endif
1687	if (fClear)
1688	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
1689	}
1690	else
1691	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
1692	}
1693
1694
1695	/**
1696	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1697	* area.
1698	*
1699	* @returns VBox status code.
1700	* @param pVCpu The cross context virtual CPU structure.
1701	* @param pVmcsInfo The VMCS info. object.
1702	* @param cMsrs The number of MSRs.
1703	*/
1704	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
1705	{
1706	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1707	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1708	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
1709	{
1710	/* Commit the MSR counts to the VMCS and update the cache. */
1711	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
1712	{
1713	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1714	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1715	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1716	AssertRCReturn(rc, rc);
1717
1718	pVmcsInfo->cEntryMsrLoad = cMsrs;
1719	pVmcsInfo->cExitMsrStore = cMsrs;
1720	pVmcsInfo->cExitMsrLoad = cMsrs;
1721	}
1722	return VINF_SUCCESS;
1723	}
1724
1725	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
1726	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1727	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1728	}
1729
1730
1731	/**
1732	* Adds a new (or updates the value of an existing) guest/host MSR
1733	* pair to be swapped during the world-switch as part of the
1734	* auto-load/store MSR area in the VMCS.
1735	*
1736	* @returns VBox status code.
1737	* @param pVCpu The cross context virtual CPU structure.
1738	* @param pVmxTransient The VMX-transient structure.
1739	* @param idMsr The MSR.
1740	* @param uGuestMsrValue Value of the guest MSR.
1741	* @param fSetReadWrite Whether to set the guest read/write access of this
1742	* MSR (thus not causing a VM-exit).
1743	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1744	* necessary.
1745	*/
1746	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
1747	bool fSetReadWrite, bool fUpdateHostMsr)
1748	{
1749	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1750	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1751	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1752	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1753	uint32_t i;
1754
1755	/* Paranoia. */
1756	Assert(pGuestMsrLoad);
1757
1758	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
1759
1760	/* Check if the MSR already exists in the VM-entry MSR-load area. */
1761	for (i = 0; i < cMsrs; i++)
1762	{
1763	if (pGuestMsrLoad[i].u32Msr == idMsr)
1764	break;
1765	}
1766
1767	bool fAdded = false;
1768	if (i == cMsrs)
1769	{
1770	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
1771	++cMsrs;
1772	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1773	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
1774
1775	/* Set the guest to read/write this MSR without causing VM-exits. */
1776	if ( fSetReadWrite
1777	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
1778	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
1779
1780	LogFlowFunc(("MSR added, cMsrs now %u\n", cMsrs));
1781	fAdded = true;
1782	}
1783
1784	/* Update the MSR value for the newly added or already existing MSR. */
1785	pGuestMsrLoad[i].u32Msr = idMsr;
1786	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
1787
1788	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
1789	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1790	{
1791	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1792	pGuestMsrStore[i].u32Msr = idMsr;
1793	pGuestMsrStore[i].u64Value = uGuestMsrValue;
1794	}
1795
1796	/* Update the corresponding slot in the host MSR area. */
1797	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1798	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
1799	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
1800	pHostMsr[i].u32Msr = idMsr;
1801
1802	/*
1803	* Only if the caller requests to update the host MSR value AND we've newly added the
1804	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
1805	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
1806	*
1807	* We do this for performance reasons since reading MSRs may be quite expensive.
1808	*/
1809	if (fAdded)
1810	{
1811	if (fUpdateHostMsr)
1812	{
1813	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1814	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1815	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
1816	}
1817	else
1818	{
1819	/* Someone else can do the work. */
1820	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
1821	}
1822	}
1823	return VINF_SUCCESS;
1824	}
1825
1826
1827	/**
1828	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1829	* auto-load/store MSR area in the VMCS.
1830	*
1831	* @returns VBox status code.
1832	* @param pVCpu The cross context virtual CPU structure.
1833	* @param pVmxTransient The VMX-transient structure.
1834	* @param idMsr The MSR.
1835	*/
1836	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
1837	{
1838	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1839	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1840	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1841	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1842
1843	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
1844
1845	for (uint32_t i = 0; i < cMsrs; i++)
1846	{
1847	/* Find the MSR. */
1848	if (pGuestMsrLoad[i].u32Msr == idMsr)
1849	{
1850	/*
1851	* If it's the last MSR, we only need to reduce the MSR count.
1852	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
1853	*/
1854	if (i < cMsrs - 1)
1855	{
1856	/* Remove it from the VM-entry MSR-load area. */
1857	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
1858	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
1859
1860	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
1861	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1862	{
1863	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1864	Assert(pGuestMsrStore[i].u32Msr == idMsr);
1865	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
1866	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
1867	}
1868
1869	/* Remove it from the VM-exit MSR-load area. */
1870	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1871	Assert(pHostMsr[i].u32Msr == idMsr);
1872	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
1873	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
1874	}
1875
1876	/* Reduce the count to reflect the removed MSR and bail. */
1877	--cMsrs;
1878	break;
1879	}
1880	}
1881
1882	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
1883	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
1884	{
1885	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1886	AssertRCReturn(rc, rc);
1887
1888	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1889	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1890	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
1891
1892	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
1893	return VINF_SUCCESS;
1894	}
1895
1896	return VERR_NOT_FOUND;
1897	}
1898
1899
1900	/**
1901	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
1902	*
1903	* @returns @c true if found, @c false otherwise.
1904	* @param pVmcsInfo The VMCS info. object.
1905	* @param idMsr The MSR to find.
1906	*/
1907	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
1908	{
1909	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1910	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
1911	Assert(pMsrs);
1912	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
1913	for (uint32_t i = 0; i < cMsrs; i++)
1914	{
1915	if (pMsrs[i].u32Msr == idMsr)
1916	return true;
1917	}
1918	return false;
1919	}
1920
1921
1922	/**
1923	* Updates the value of all host MSRs in the VM-exit MSR-load area.
1924	*
1925	* @param pVCpu The cross context virtual CPU structure.
1926	* @param pVmcsInfo The VMCS info. object.
1927	*
1928	* @remarks No-long-jump zone!!!
1929	*/
1930	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
1931	{
1932	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1933
1934	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1935	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
1936	Assert(pHostMsrLoad);
1937	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
1938	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
1939	for (uint32_t i = 0; i < cMsrs; i++)
1940	{
1941	/*
1942	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1943	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1944	*/
1945	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
1946	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
1947	else
1948	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
1949	}
1950	}
1951
1952
1953	/**
1954	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1955	* perform lazy restoration of the host MSRs while leaving VT-x.
1956	*
1957	* @param pVCpu The cross context virtual CPU structure.
1958	*
1959	* @remarks No-long-jump zone!!!
1960	*/
1961	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1962	{
1963	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1964
1965	/*
1966	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
1967	*/
1968	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1969	{
1970	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1971	#if HC_ARCH_BITS == 64
1972	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1973	{
1974	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
1975	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
1976	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
1977	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1978	}
1979	#endif
1980	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1981	}
1982	}
1983
1984
1985	/**
1986	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1987	* lazily while leaving VT-x.
1988	*
1989	* @returns true if it does, false otherwise.
1990	* @param pVCpu The cross context virtual CPU structure.
1991	* @param idMsr The MSR to check.
1992	*/
1993	static bool hmR0VmxIsLazyGuestMsr(PCVMCPU pVCpu, uint32_t idMsr)
1994	{
1995	NOREF(pVCpu);
1996	#if HC_ARCH_BITS == 64
1997	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1998	{
1999	switch (idMsr)
2000	{
2001	case MSR_K8_LSTAR:
2002	case MSR_K6_STAR:
2003	case MSR_K8_SF_MASK:
2004	case MSR_K8_KERNEL_GS_BASE:
2005	return true;
2006	}
2007	}
2008	#else
2009	RT_NOREF(pVCpu, idMsr);
2010	#endif
2011	return false;
2012	}
2013
2014
2015	/**
2016	* Loads a set of guests MSRs to allow read/passthru to the guest.
2017	*
2018	* The name of this function is slightly confusing. This function does NOT
2019	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
2020	* common prefix for functions dealing with "lazy restoration" of the shared
2021	* MSRs.
2022	*
2023	* @param pVCpu The cross context virtual CPU structure.
2024	*
2025	* @remarks No-long-jump zone!!!
2026	*/
2027	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
2028	{
2029	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2030	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2031
2032	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2033	#if HC_ARCH_BITS == 64
2034	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2035	{
2036	/*
2037	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
2038	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
2039	* we can skip a few MSR writes.
2040	*
2041	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
2042	* guest MSR values in the guest-CPU context might be different to what's currently
2043	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
2044	* CPU, see @bugref{8728}.
2045	*/
2046	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2047	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2048	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
2049	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
2050	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
2051	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
2052	{
2053	#ifdef VBOX_STRICT
2054	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
2055	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
2056	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
2057	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
2058	#endif
2059	}
2060	else
2061	{
2062	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2063	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2064	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2065	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2066	}
2067	}
2068	#endif
2069	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2070	}
2071
2072
2073	/**
2074	* Performs lazy restoration of the set of host MSRs if they were previously
2075	* loaded with guest MSR values.
2076	*
2077	* @param pVCpu The cross context virtual CPU structure.
2078	*
2079	* @remarks No-long-jump zone!!!
2080	* @remarks The guest MSRs should have been saved back into the guest-CPU
2081	* context by hmR0VmxImportGuestState()!!!
2082	*/
2083	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2084	{
2085	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2086	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2087
2088	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2089	{
2090	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2091	#if HC_ARCH_BITS == 64
2092	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2093	{
2094	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2095	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2096	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2097	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2098	}
2099	#endif
2100	}
2101	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2102	}
2103
2104
2105	/**
2106	* Verifies that our cached values of the VMCS fields are all consistent with
2107	* what's actually present in the VMCS.
2108	*
2109	* @returns VBox status code.
2110	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2111	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2112	* VMCS content. HMCPU error-field is
2113	* updated, see VMX_VCI_XXX.
2114	* @param pVCpu The cross context virtual CPU structure.
2115	* @param pVmcsInfo The VMCS info. object.
2116	*/
2117	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2118	{
2119	uint32_t u32Val;
2120	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2121	AssertRCReturn(rc, rc);
2122	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2123	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32EntryCtls, u32Val),
2124	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2125	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2126
2127	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2128	AssertRCReturn(rc, rc);
2129	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2130	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ExitCtls, u32Val),
2131	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2132	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2133
2134	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2135	AssertRCReturn(rc, rc);
2136	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2137	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32PinCtls, u32Val),
2138	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2139	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2140
2141	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2142	AssertRCReturn(rc, rc);
2143	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2144	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls, u32Val),
2145	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2146	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2147
2148	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2149	{
2150	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2151	AssertRCReturn(rc, rc);
2152	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2153	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls2, u32Val),
2154	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2155	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2156	}
2157
2158	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2159	AssertRCReturn(rc, rc);
2160	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2161	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32XcptBitmap, u32Val),
2162	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2163	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2164
2165	uint64_t u64Val;
2166	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2167	AssertRCReturn(rc, rc);
2168	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2169	("Cache=%#RX64 VMCS=%#RX64\n", pVmcsInfo->u64TscOffset, u64Val),
2170	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2171	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2172
2173	return VINF_SUCCESS;
2174	}
2175
2176
2177	#ifdef VBOX_STRICT
2178	/**
2179	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2180	*
2181	* @param pVCpu The cross context virtual CPU structure.
2182	* @param pVmcsInfo The VMCS info. object.
2183	*/
2184	static void hmR0VmxCheckHostEferMsr(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2185	{
2186	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2187
2188	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2189	{
2190	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2191	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2192	uint64_t uVmcsEferMsrVmcs;
2193	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2194	AssertRC(rc);
2195
2196	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2197	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2198	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2199	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2200	}
2201	}
2202
2203
2204	/**
2205	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2206	* VMCS are correct.
2207	*
2208	* @param pVCpu The cross context virtual CPU structure.
2209	* @param pVmcsInfo The VMCS info. object.
2210	*/
2211	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2212	{
2213	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2214
2215	/* Read the various MSR-area counts from the VMCS. */
2216	uint32_t cEntryLoadMsrs;
2217	uint32_t cExitStoreMsrs;
2218	uint32_t cExitLoadMsrs;
2219	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2220	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2221	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2222
2223	/* Verify all the MSR counts are the same. */
2224	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2225	Assert(cExitStoreMsrs == cExitLoadMsrs);
2226	uint32_t const cMsrs = cExitLoadMsrs;
2227
2228	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2229	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2230
2231	/* Verify the MSR counts are within the allocated page size. */
2232	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2233
2234	/* Verify the relevant contents of the MSR areas match. */
2235	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2236	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2237	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2238	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2239	for (uint32_t i = 0; i < cMsrs; i++)
2240	{
2241	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2242	if (fSeparateExitMsrStorePage)
2243	{
2244	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2245	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2246	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2247	}
2248
2249	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2250	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2251	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2252
2253	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2254	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2255	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2256	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2257
2258	/* Verify that cached host EFER MSR matches what's loaded the CPU. */
2259	bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2260	if (fIsEferMsr)
2261	{
2262	AssertMsgReturnVoid(u64Msr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2263	("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2264	pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64Msr, cMsrs));
2265	}
2266
2267	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2268	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2269	{
2270	uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2271	if (fIsEferMsr)
2272	{
2273	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2274	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2275	}
2276	else
2277	{
2278	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2279	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2280	}
2281	}
2282
2283	/* Move to the next MSR. */
2284	pHostMsrLoad++;
2285	pGuestMsrLoad++;
2286	pGuestMsrStore++;
2287	}
2288	}
2289	#endif /* VBOX_STRICT */
2290
2291
2292	/**
2293	* Flushes the TLB using EPT.
2294	*
2295	* @returns VBox status code.
2296	* @param pVCpu The cross context virtual CPU structure of the calling
2297	* EMT. Can be NULL depending on @a enmTlbFlush.
2298	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2299	* enmTlbFlush.
2300	* @param enmTlbFlush Type of flush.
2301	*
2302	* @remarks Caller is responsible for making sure this function is called only
2303	* when NestedPaging is supported and providing @a enmTlbFlush that is
2304	* supported by the CPU.
2305	* @remarks Can be called with interrupts disabled.
2306	*/
2307	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2308	{
2309	uint64_t au64Descriptor[2];
2310	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2311	au64Descriptor[0] = 0;
2312	else
2313	{
2314	Assert(pVCpu);
2315	Assert(pVmcsInfo);
2316	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2317	}
2318	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2319
2320	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2321	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2322
2323	if ( RT_SUCCESS(rc)
2324	&& pVCpu)
2325	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2326	}
2327
2328
2329	/**
2330	* Flushes the TLB using VPID.
2331	*
2332	* @returns VBox status code.
2333	* @param pVCpu The cross context virtual CPU structure of the calling
2334	* EMT. Can be NULL depending on @a enmTlbFlush.
2335	* @param enmTlbFlush Type of flush.
2336	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2337	* on @a enmTlbFlush).
2338	*
2339	* @remarks Can be called with interrupts disabled.
2340	*/
2341	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2342	{
2343	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2344
2345	uint64_t au64Descriptor[2];
2346	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2347	{
2348	au64Descriptor[0] = 0;
2349	au64Descriptor[1] = 0;
2350	}
2351	else
2352	{
2353	AssertPtr(pVCpu);
2354	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2355	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2356	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2357	au64Descriptor[1] = GCPtr;
2358	}
2359
2360	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2361	AssertMsg(rc == VINF_SUCCESS,
2362	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2363
2364	if ( RT_SUCCESS(rc)
2365	&& pVCpu)
2366	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2367	NOREF(rc);
2368	}
2369
2370
2371	/**
2372	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2373	* otherwise there is nothing really to invalidate.
2374	*
2375	* @returns VBox status code.
2376	* @param pVCpu The cross context virtual CPU structure.
2377	* @param GCVirt Guest virtual address of the page to invalidate.
2378	*/
2379	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2380	{
2381	AssertPtr(pVCpu);
2382	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2383
2384	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2385	{
2386	/*
2387	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2388	* the EPT case. See @bugref{6043} and @bugref{6177}.
2389	*
2390	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2391	* as this function maybe called in a loop with individual addresses.
2392	*/
2393	PVM pVM = pVCpu->CTX_SUFF(pVM);
2394	if (pVM->hm.s.vmx.fVpid)
2395	{
2396	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2397
2398	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2399	/*
2400	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2401	* where executing INVVPID outside 64-bit mode does not flush translations of
2402	* 64-bit linear addresses, see @bugref{6208#c72}.
2403	*/
2404	if (RT_HI_U32(GCVirt))
2405	fVpidFlush = false;
2406	#endif
2407
2408	if (fVpidFlush)
2409	{
2410	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2411	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2412	}
2413	else
2414	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2415	}
2416	else if (pVM->hm.s.fNestedPaging)
2417	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2418	}
2419
2420	return VINF_SUCCESS;
2421	}
2422
2423
2424	/**
2425	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2426	* case where neither EPT nor VPID is supported by the CPU.
2427	*
2428	* @param pHostCpu The HM physical-CPU structure.
2429	* @param pVCpu The cross context virtual CPU structure.
2430	*
2431	* @remarks Called with interrupts disabled.
2432	*/
2433	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2434	{
2435	AssertPtr(pVCpu);
2436	AssertPtr(pHostCpu);
2437
2438	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2439
2440	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2441	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2442	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2443	pVCpu->hm.s.fForceTLBFlush = false;
2444	return;
2445	}
2446
2447
2448	/**
2449	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2450	*
2451	* @param pHostCpu The HM physical-CPU structure.
2452	* @param pVCpu The cross context virtual CPU structure.
2453	* @param pVmcsInfo The VMCS info. object.
2454	*
2455	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2456	* nomenclature. The reason is, to avoid confusion in compare statements
2457	* since the host-CPU copies are named "ASID".
2458	*
2459	* @remarks Called with interrupts disabled.
2460	*/
2461	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2462	{
2463	#ifdef VBOX_WITH_STATISTICS
2464	bool fTlbFlushed = false;
2465	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2466	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2467	if (!fTlbFlushed) \
2468	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2469	} while (0)
2470	#else
2471	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2472	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2473	#endif
2474
2475	AssertPtr(pVCpu);
2476	AssertPtr(pHostCpu);
2477	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2478
2479	PVM pVM = pVCpu->CTX_SUFF(pVM);
2480	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2481	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2482	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2483
2484	/*
2485	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2486	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2487	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2488	* cannot reuse the current ASID anymore.
2489	*/
2490	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2491	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2492	{
2493	++pHostCpu->uCurrentAsid;
2494	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2495	{
2496	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2497	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2498	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2499	}
2500
2501	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2502	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2503	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2504
2505	/*
2506	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2507	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2508	*/
2509	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2510	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2511	HMVMX_SET_TAGGED_TLB_FLUSHED();
2512	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2513	}
2514	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2515	{
2516	/*
2517	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2518	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2519	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2520	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2521	* mappings, see @bugref{6568}.
2522	*
2523	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2524	*/
2525	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2526	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2527	HMVMX_SET_TAGGED_TLB_FLUSHED();
2528	}
2529
2530	pVCpu->hm.s.fForceTLBFlush = false;
2531	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2532
2533	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
2534	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
2535	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2536	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2537	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2538	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2539	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2540	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2541	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2542
2543	/* Update VMCS with the VPID. */
2544	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2545	AssertRC(rc);
2546
2547	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2548	}
2549
2550
2551	/**
2552	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2553	*
2554	* @param pHostCpu The HM physical-CPU structure.
2555	* @param pVCpu The cross context virtual CPU structure.
2556	* @param pVmcsInfo The VMCS info. object.
2557	*
2558	* @remarks Called with interrupts disabled.
2559	*/
2560	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2561	{
2562	AssertPtr(pVCpu);
2563	AssertPtr(pHostCpu);
2564	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2565	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2566	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2567
2568	/*
2569	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2570	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2571	*/
2572	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2573	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2574	{
2575	pVCpu->hm.s.fForceTLBFlush = true;
2576	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2577	}
2578
2579	/* Check for explicit TLB flushes. */
2580	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2581	{
2582	pVCpu->hm.s.fForceTLBFlush = true;
2583	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2584	}
2585
2586	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2587	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2588
2589	if (pVCpu->hm.s.fForceTLBFlush)
2590	{
2591	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2592	pVCpu->hm.s.fForceTLBFlush = false;
2593	}
2594	}
2595
2596
2597	/**
2598	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2599	*
2600	* @param pHostCpu The HM physical-CPU structure.
2601	* @param pVCpu The cross context virtual CPU structure.
2602	*
2603	* @remarks Called with interrupts disabled.
2604	*/
2605	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2606	{
2607	AssertPtr(pVCpu);
2608	AssertPtr(pHostCpu);
2609	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2610	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2611	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2612
2613	/*
2614	* Force a TLB flush for the first world switch if the current CPU differs from the one we
2615	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2616	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2617	* cannot reuse the current ASID anymore.
2618	*/
2619	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2620	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2621	{
2622	pVCpu->hm.s.fForceTLBFlush = true;
2623	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2624	}
2625
2626	/* Check for explicit TLB flushes. */
2627	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2628	{
2629	/*
2630	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2631	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2632	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
2633	* include fExplicitFlush's too) - an obscure corner case.
2634	*/
2635	pVCpu->hm.s.fForceTLBFlush = true;
2636	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2637	}
2638
2639	PVM pVM = pVCpu->CTX_SUFF(pVM);
2640	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2641	if (pVCpu->hm.s.fForceTLBFlush)
2642	{
2643	++pHostCpu->uCurrentAsid;
2644	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2645	{
2646	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2647	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2648	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2649	}
2650
2651	pVCpu->hm.s.fForceTLBFlush = false;
2652	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2653	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2654	if (pHostCpu->fFlushAsidBeforeUse)
2655	{
2656	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2657	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2658	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2659	{
2660	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2661	pHostCpu->fFlushAsidBeforeUse = false;
2662	}
2663	else
2664	{
2665	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2666	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2667	}
2668	}
2669	}
2670
2671	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2672	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2673	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2674	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2675	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2676	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2677	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2678
2679	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2680	AssertRC(rc);
2681	}
2682
2683
2684	/**
2685	* Flushes the guest TLB entry based on CPU capabilities.
2686	*
2687	* @param pHostCpu The HM physical-CPU structure.
2688	* @param pVCpu The cross context virtual CPU structure.
2689	* @param pVmcsInfo The VMCS info. object.
2690	*
2691	* @remarks Called with interrupts disabled.
2692	*/
2693	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2694	{
2695	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2696	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2697	#endif
2698	PVM pVM = pVCpu->CTX_SUFF(pVM);
2699	switch (pVM->hm.s.vmx.enmTlbFlushType)
2700	{
2701	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
2702	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
2703	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
2704	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
2705	default:
2706	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2707	break;
2708	}
2709	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2710	}
2711
2712
2713	/**
2714	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2715	* TLB entries from the host TLB before VM-entry.
2716	*
2717	* @returns VBox status code.
2718	* @param pVM The cross context VM structure.
2719	*/
2720	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2721	{
2722	/*
2723	* Determine optimal flush type for nested paging.
2724	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
2725	* unrestricted guest execution (see hmR3InitFinalizeR0()).
2726	*/
2727	if (pVM->hm.s.fNestedPaging)
2728	{
2729	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2730	{
2731	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2732	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2733	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2734	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2735	else
2736	{
2737	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2738	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2739	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2740	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2741	}
2742
2743	/* Make sure the write-back cacheable memory type for EPT is supported. */
2744	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2745	{
2746	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2747	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2748	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2749	}
2750
2751	/* EPT requires a page-walk length of 4. */
2752	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2753	{
2754	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2755	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2756	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2757	}
2758	}
2759	else
2760	{
2761	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2762	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2763	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2764	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2765	}
2766	}
2767
2768	/*
2769	* Determine optimal flush type for VPID.
2770	*/
2771	if (pVM->hm.s.vmx.fVpid)
2772	{
2773	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2774	{
2775	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2776	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2777	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2778	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2779	else
2780	{
2781	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2782	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2783	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2784	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2785	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2786	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2787	pVM->hm.s.vmx.fVpid = false;
2788	}
2789	}
2790	else
2791	{
2792	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2793	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2794	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2795	pVM->hm.s.vmx.fVpid = false;
2796	}
2797	}
2798
2799	/*
2800	* Setup the handler for flushing tagged-TLBs.
2801	*/
2802	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2803	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2804	else if (pVM->hm.s.fNestedPaging)
2805	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2806	else if (pVM->hm.s.vmx.fVpid)
2807	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2808	else
2809	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2810	return VINF_SUCCESS;
2811	}
2812
2813
2814	/**
2815	* Sets up the virtual-APIC page address for the VMCS.
2816	*
2817	* @returns VBox status code.
2818	* @param pVCpu The cross context virtual CPU structure.
2819	* @param pVmcsInfo The VMCS info. object.
2820	*/
2821	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2822	{
2823	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2824	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
2825	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
2826	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2827	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
2828	}
2829
2830
2831	/**
2832	* Sets up the MSR-bitmap address for the VMCS.
2833	*
2834	* @returns VBox status code.
2835	* @param pVCpu The cross context virtual CPU structure.
2836	* @param pVmcsInfo The VMCS info. object.
2837	*/
2838	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2839	{
2840	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2841	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
2842	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
2843	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2844	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
2845	}
2846
2847
2848	/**
2849	* Sets up the APIC-access page address for the VMCS.
2850	*
2851	* @returns VBox status code.
2852	* @param pVCpu The cross context virtual CPU structure.
2853	*/
2854	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
2855	{
2856	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
2857	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
2858	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2859	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
2860	}
2861
2862
2863	/**
2864	* Sets up the VMCS link pointer for the VMCS.
2865	*
2866	* @returns VBox status code.
2867	* @param pVCpu The cross context virtual CPU structure.
2868	* @param pVmcsInfo The VMCS info. object.
2869	*/
2870	DECLINLINE(int) hmR0VmxSetupVmcsLinkPtr(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2871	{
2872	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2873	uint64_t const u64VmcsLinkPtr = pVmcsInfo->u64VmcsLinkPtr;
2874	Assert(u64VmcsLinkPtr == UINT64_C(0xffffffffffffffff)); /* Bits 63:0 MB1. */
2875	return VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, u64VmcsLinkPtr);
2876	}
2877
2878
2879	/**
2880	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
2881	* in the VMCS.
2882	*
2883	* @returns VBox status code.
2884	* @param pVCpu The cross context virtual CPU structure.
2885	* @param pVmcsInfo The VMCS info. object.
2886	*/
2887	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2888	{
2889	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2890
2891	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
2892	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
2893	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2894
2895	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
2896	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
2897	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
2898
2899	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
2900	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
2901	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2902
2903	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
2904	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
2905	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
2906	AssertRCReturn(rc, rc);
2907	return VINF_SUCCESS;
2908	}
2909
2910
2911	/**
2912	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
2913	*
2914	* @param pVCpu The cross context virtual CPU structure.
2915	* @param pVmcsInfo The VMCS info. object.
2916	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2917	*/
2918	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2919	{
2920	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
2921
2922	/*
2923	* The guest can access the following MSRs (read, write) without causing
2924	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
2925	*/
2926	PVM pVM = pVCpu->CTX_SUFF(pVM);
2927	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
2928	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
2929	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
2930	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2931	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
2932
2933	/*
2934	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
2935	* associated with then. We never need to intercept access (writes need to be
2936	* executed without causing a VM-exit, reads will #GP fault anyway).
2937	*
2938	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
2939	* read/write them. We swap the the guest/host MSR value using the
2940	* auto-load/store MSR area.
2941	*/
2942	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2943	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
2944	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
2945	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
2946	if (pVM->cpum.ro.GuestFeatures.fIbrs)
2947	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
2948
2949	#if HC_ARCH_BITS == 64
2950	/*
2951	* Allow full read/write access for the following MSRs (mandatory for VT-x)
2952	* required for 64-bit guests.
2953	*/
2954	if (pVM->hm.s.fAllow64BitGuests)
2955	{
2956	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
2957	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
2958	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
2959	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2960	}
2961	#endif
2962
2963	/*
2964	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
2965	*/
2966	#ifdef VBOX_STRICT
2967	Assert(pVmcsInfo->pvMsrBitmap);
2968	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
2969	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
2970	#endif
2971	}
2972
2973
2974	/**
2975	* Sets up pin-based VM-execution controls in the VMCS.
2976	*
2977	* @returns VBox status code.
2978	* @param pVCpu The cross context virtual CPU structure.
2979	* @param pVmcsInfo The VMCS info. object.
2980	*/
2981	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2982	{
2983	PVM pVM = pVCpu->CTX_SUFF(pVM);
2984	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
2985	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2986
2987	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2988	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2989
2990	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2991	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2992
2993	/* Enable the VMX-preemption timer. */
2994	if (pVM->hm.s.vmx.fUsePreemptTimer)
2995	{
2996	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2997	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
2998	}
2999
3000	#if 0
3001	/* Enable posted-interrupt processing. */
3002	if (pVM->hm.s.fPostedIntrs)
3003	{
3004	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
3005	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
3006	fVal \|= VMX_PIN_CTLS_POSTED_INT;
3007	}
3008	#endif
3009
3010	if ((fVal & fZap) != fVal)
3011	{
3012	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3013	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
3014	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
3015	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3016	}
3017
3018	/* Commit it to the VMCS and update our cache. */
3019	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
3020	AssertRCReturn(rc, rc);
3021	pVmcsInfo->u32PinCtls = fVal;
3022
3023	return VINF_SUCCESS;
3024	}
3025
3026
3027	/**
3028	* Sets up secondary processor-based VM-execution controls in the VMCS.
3029	*
3030	* @returns VBox status code.
3031	* @param pVCpu The cross context virtual CPU structure.
3032	* @param pVmcsInfo The VMCS info. object.
3033	*/
3034	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3035	{
3036	PVM pVM = pVCpu->CTX_SUFF(pVM);
3037	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
3038	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3039
3040	/* WBINVD causes a VM-exit. */
3041	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
3042	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
3043
3044	/* Enable EPT (aka nested-paging). */
3045	if (pVM->hm.s.fNestedPaging)
3046	fVal \|= VMX_PROC_CTLS2_EPT;
3047
3048	/* Enable the INVPCID instruction if supported by the hardware and we expose
3049	it to the guest. Without this, guest executing INVPCID would cause a #UD. */
3050	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
3051	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
3052	fVal \|= VMX_PROC_CTLS2_INVPCID;
3053
3054	/* Enable VPID. */
3055	if (pVM->hm.s.vmx.fVpid)
3056	fVal \|= VMX_PROC_CTLS2_VPID;
3057
3058	/* Enable unrestricted guest execution. */
3059	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3060	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3061
3062	#if 0
3063	if (pVM->hm.s.fVirtApicRegs)
3064	{
3065	/* Enable APIC-register virtualization. */
3066	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3067	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3068
3069	/* Enable virtual-interrupt delivery. */
3070	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3071	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3072	}
3073	#endif
3074
3075	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
3076	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3077	* done dynamically. */
3078	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3079	{
3080	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3081	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3082	AssertRCReturn(rc, rc);
3083	}
3084
3085	/* Enable the RDTSCP instruction if supported by the hardware and we expose
3086	it to the guest. Without this, guest executing RDTSCP would cause a #UD. */
3087	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
3088	&& pVM->cpum.ro.GuestFeatures.fRdTscP)
3089	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3090
3091	/* Enable Pause-Loop exiting. */
3092	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
3093	&& pVM->hm.s.vmx.cPleGapTicks
3094	&& pVM->hm.s.vmx.cPleWindowTicks)
3095	{
3096	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3097
3098	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3099	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3100	AssertRCReturn(rc, rc);
3101	}
3102
3103	if ((fVal & fZap) != fVal)
3104	{
3105	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3106	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3107	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3108	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3109	}
3110
3111	/* Commit it to the VMCS and update our cache. */
3112	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3113	AssertRCReturn(rc, rc);
3114	pVmcsInfo->u32ProcCtls2 = fVal;
3115
3116	return VINF_SUCCESS;
3117	}
3118
3119
3120	/**
3121	* Sets up processor-based VM-execution controls in the VMCS.
3122	*
3123	* @returns VBox status code.
3124	* @param pVCpu The cross context virtual CPU structure.
3125	* @param pVmcsInfo The VMCS info. object.
3126	*/
3127	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3128	{
3129	PVM pVM = pVCpu->CTX_SUFF(pVM);
3130
3131	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3132	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3133
3134	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3135	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3136	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3137	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3138	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3139	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3140	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3141
3142	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3143	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3144	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3145	{
3146	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3147	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3148	}
3149
3150	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3151	if (!pVM->hm.s.fNestedPaging)
3152	{
3153	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3154	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3155	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3156	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3157	}
3158
3159	/* Use TPR shadowing if supported by the CPU. */
3160	if ( PDMHasApic(pVM)
3161	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3162	{
3163	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3164	/* CR8 writes cause a VM-exit based on TPR threshold. */
3165	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3166	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3167	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3168	AssertRCReturn(rc, rc);
3169	}
3170	else
3171	{
3172	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3173	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3174	if (pVM->hm.s.fAllow64BitGuests)
3175	{
3176	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3177	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3178	}
3179	}
3180
3181	/* Use MSR-bitmaps if supported by the CPU. */
3182	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3183	{
3184	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3185	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3186	AssertRCReturn(rc, rc);
3187	}
3188
3189	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3190	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3191	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3192
3193	if ((fVal & fZap) != fVal)
3194	{
3195	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3196	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3197	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3198	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3199	}
3200
3201	/* Commit it to the VMCS and update our cache. */
3202	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3203	AssertRCReturn(rc, rc);
3204	pVmcsInfo->u32ProcCtls = fVal;
3205
3206	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3207	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3208	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo, false /* fIsNstGstVmcs */);
3209
3210	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3211	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3212	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3213
3214	/* Sanity check, should not really happen. */
3215	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3216	{ /* likely */ }
3217	else
3218	{
3219	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3220	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3221	}
3222
3223	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3224	return VINF_SUCCESS;
3225	}
3226
3227
3228	/**
3229	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3230	* Processor-based VM-execution) control fields in the VMCS.
3231	*
3232	* @returns VBox status code.
3233	* @param pVCpu The cross context virtual CPU structure.
3234	* @param pVmcsInfo The VMCS info. object.
3235	*/
3236	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3237	{
3238	/* Set the auto-load/store MSR area addresses in the VMCS. */
3239	int rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3240	if (RT_SUCCESS(rc))
3241	{
3242	/* Set the VMCS link pointer in the VMCS. */
3243	rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3244	if (RT_SUCCESS(rc))
3245	{
3246	/* Set the CR0/CR4 guest/host mask. */
3247	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3248	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3249	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3250	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3251	if (RT_SUCCESS(rc))
3252	{
3253	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3254	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3255	return VINF_SUCCESS;
3256	}
3257	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3258	}
3259	else
3260	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3261	}
3262	else
3263	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3264	return rc;
3265	}
3266
3267
3268	/**
3269	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3270	*
3271	* We shall setup those exception intercepts that don't change during the
3272	* lifetime of the VM here. The rest are done dynamically while loading the
3273	* guest state.
3274	*
3275	* @returns VBox status code.
3276	* @param pVCpu The cross context virtual CPU structure.
3277	* @param pVmcsInfo The VMCS info. object.
3278	*/
3279	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3280	{
3281	/*
3282	* The following exceptions are always intercepted:
3283	*
3284	* #AC - To prevent the guest from hanging the CPU.
3285	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3286	* recursive #DBs can cause a CPU hang.
3287	* #PF - To sync our shadow page tables when nested-paging is not used.
3288	*/
3289	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3290	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3291	\| RT_BIT(X86_XCPT_DB)
3292	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3293
3294	/* Commit it to the VMCS. */
3295	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3296	AssertRCReturn(rc, rc);
3297
3298	/* Update our cache of the exception bitmap. */
3299	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3300	return VINF_SUCCESS;
3301	}
3302
3303
3304	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3305	/**
3306	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3307	*
3308	* @returns VBox status code.
3309	* @param pVCpu The cross context virtual CPU structure.
3310	* @param pVmcsInfo The VMCS info. object.
3311	*/
3312	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3313	{
3314	PVM pVM = pVCpu->CTX_SUFF(pVM);
3315	int rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3316	if (RT_SUCCESS(rc))
3317	{
3318	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3319	if (RT_SUCCESS(rc))
3320	{
3321	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3322	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3323	if (RT_SUCCESS(rc))
3324	{
3325	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3326	rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3327	if (RT_SUCCESS(rc))
3328	return VINF_SUCCESS;
3329
3330	LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3331	}
3332	else
3333	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3334	}
3335	else
3336	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3337	}
3338	else
3339	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3340
3341	return rc;
3342	}
3343	#endif
3344
3345
3346	/**
3347	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3348	* VMX.
3349	*
3350	* @returns VBox status code.
3351	* @param pVCpu The cross context virtual CPU structure.
3352	* @param pVmcsInfo The VMCS info. object.
3353	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3354	*/
3355	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3356	{
3357	Assert(pVmcsInfo);
3358	Assert(pVmcsInfo->pvVmcs);
3359	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3360
3361	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3362	PVM pVM = pVCpu->CTX_SUFF(pVM);
3363	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3364	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3365
3366	LogFlowFunc(("\n"));
3367
3368	/*
3369	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3370	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3371	*/
3372	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3373	if (RT_SUCCESS(rc))
3374	{
3375	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3376	if (RT_SUCCESS(rc))
3377	{
3378	if (!fIsNstGstVmcs)
3379	{
3380	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3381	if (RT_SUCCESS(rc))
3382	{
3383	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
3384	if (RT_SUCCESS(rc))
3385	{
3386	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
3387	if (RT_SUCCESS(rc))
3388	{
3389	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
3390	if (RT_SUCCESS(rc))
3391	{ /* likely */ }
3392	else
3393	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
3394	}
3395	else
3396	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
3397	}
3398	else
3399	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
3400	}
3401	else
3402	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
3403	}
3404	else
3405	{
3406	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3407	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
3408	if (RT_SUCCESS(rc))
3409	{ /* likely */ }
3410	else
3411	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
3412	#else
3413	AssertFailed();
3414	#endif
3415	}
3416	}
3417	else
3418	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
3419	}
3420	else
3421	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
3422
3423	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
3424	if (RT_SUCCESS(rc))
3425	{
3426	rc = hmR0VmxClearVmcs(pVmcsInfo);
3427	if (RT_SUCCESS(rc))
3428	{ /* likely */ }
3429	else
3430	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
3431	}
3432
3433	/*
3434	* Update the last-error record both for failures and success, so we
3435	* can propagate the status code back to ring-3 for diagnostics.
3436	*/
3437	hmR0VmxUpdateErrorRecord(pVCpu, rc);
3438	NOREF(pszVmcs);
3439	return rc;
3440	}
3441
3442
3443	/**
3444	* Does global VT-x initialization (called during module initialization).
3445	*
3446	* @returns VBox status code.
3447	*/
3448	VMMR0DECL(int) VMXR0GlobalInit(void)
3449	{
3450	#ifdef HMVMX_USE_FUNCTION_TABLE
3451	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
3452	# ifdef VBOX_STRICT
3453	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
3454	Assert(g_apfnVMExitHandlers[i]);
3455	# endif
3456	#endif
3457	return VINF_SUCCESS;
3458	}
3459
3460
3461	/**
3462	* Does global VT-x termination (called during module termination).
3463	*/
3464	VMMR0DECL(void) VMXR0GlobalTerm()
3465	{
3466	/* Nothing to do currently. */
3467	}
3468
3469
3470	/**
3471	* Sets up and activates VT-x on the current CPU.
3472	*
3473	* @returns VBox status code.
3474	* @param pHostCpu The HM physical-CPU structure.
3475	* @param pVM The cross context VM structure. Can be
3476	* NULL after a host resume operation.
3477	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
3478	* fEnabledByHost is @c true).
3479	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
3480	* @a fEnabledByHost is @c true).
3481	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
3482	* enable VT-x on the host.
3483	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
3484	*/
3485	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
3486	PCSUPHWVIRTMSRS pHwvirtMsrs)
3487	{
3488	Assert(pHostCpu);
3489	Assert(pHwvirtMsrs);
3490	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3491
3492	/* Enable VT-x if it's not already enabled by the host. */
3493	if (!fEnabledByHost)
3494	{
3495	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
3496	if (RT_FAILURE(rc))
3497	return rc;
3498	}
3499
3500	/*
3501	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
3502	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
3503	* invalidated when flushing by VPID.
3504	*/
3505	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3506	{
3507	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
3508	pHostCpu->fFlushAsidBeforeUse = false;
3509	}
3510	else
3511	pHostCpu->fFlushAsidBeforeUse = true;
3512
3513	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
3514	++pHostCpu->cTlbFlushes;
3515
3516	return VINF_SUCCESS;
3517	}
3518
3519
3520	/**
3521	* Deactivates VT-x on the current CPU.
3522	*
3523	* @returns VBox status code.
3524	* @param pvCpuPage Pointer to the VMXON region.
3525	* @param HCPhysCpuPage Physical address of the VMXON region.
3526	*
3527	* @remarks This function should never be called when SUPR0EnableVTx() or
3528	* similar was used to enable VT-x on the host.
3529	*/
3530	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
3531	{
3532	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
3533
3534	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3535	return hmR0VmxLeaveRootMode();
3536	}
3537
3538
3539	/**
3540	* Does per-VM VT-x initialization.
3541	*
3542	* @returns VBox status code.
3543	* @param pVM The cross context VM structure.
3544	*/
3545	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
3546	{
3547	LogFlowFunc(("pVM=%p\n", pVM));
3548
3549	int rc = hmR0VmxStructsAlloc(pVM);
3550	if (RT_FAILURE(rc))
3551	{
3552	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
3553	return rc;
3554	}
3555
3556	return VINF_SUCCESS;
3557	}
3558
3559
3560	/**
3561	* Does per-VM VT-x termination.
3562	*
3563	* @returns VBox status code.
3564	* @param pVM The cross context VM structure.
3565	*/
3566	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
3567	{
3568	LogFlowFunc(("pVM=%p\n", pVM));
3569
3570	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3571	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
3572	{
3573	Assert(pVM->hm.s.vmx.pvScratch);
3574	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
3575	}
3576	#endif
3577	hmR0VmxStructsFree(pVM);
3578	return VINF_SUCCESS;
3579	}
3580
3581
3582	/**
3583	* Sets up the VM for execution using hardware-assisted VMX.
3584	* This function is only called once per-VM during initialization.
3585	*
3586	* @returns VBox status code.
3587	* @param pVM The cross context VM structure.
3588	*/
3589	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
3590	{
3591	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
3592	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3593
3594	LogFlowFunc(("pVM=%p\n", pVM));
3595
3596	/*
3597	* At least verify if VMX is enabled, since we can't check if we're in
3598	* VMX root mode or not without causing a #GP.
3599	*/
3600	RTCCUINTREG const uHostCR4 = ASMGetCR4();
3601	if (RT_LIKELY(uHostCR4 & X86_CR4_VMXE))
3602	{ /* likely */ }
3603	else
3604	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
3605
3606	/*
3607	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
3608	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
3609	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
3610	*/
3611	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3612	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
3613	\|\| !pVM->hm.s.vmx.pRealModeTSS))
3614	{
3615	LogRelFunc(("Invalid real-on-v86 state.\n"));
3616	return VERR_INTERNAL_ERROR;
3617	}
3618
3619	/* Initialize these always, see hmR3InitFinalizeR0().*/
3620	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
3621	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
3622
3623	/* Setup the tagged-TLB flush handlers. */
3624	int rc = hmR0VmxSetupTaggedTlb(pVM);
3625	if (RT_FAILURE(rc))
3626	{
3627	LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
3628	return rc;
3629	}
3630
3631	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
3632	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
3633	#if HC_ARCH_BITS == 64
3634	if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
3635	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
3636	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
3637	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
3638	#endif
3639
3640	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3641	{
3642	PVMCPU pVCpu = &pVM->aCpus[idCpu];
3643	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
3644
3645	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
3646	if (RT_SUCCESS(rc))
3647	{
3648	#if HC_ARCH_BITS == 32
3649	hmR0VmxInitVmcsReadCache(pVCpu);
3650	#endif
3651	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3652	if (pVM->cpum.ro.GuestFeatures.fVmx)
3653	{
3654	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
3655	if (RT_SUCCESS(rc))
3656	{ /* likely */ }
3657	else
3658	{
3659	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
3660	return rc;
3661	}
3662	}
3663	#endif
3664	}
3665	else
3666	{
3667	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
3668	return rc;
3669	}
3670	}
3671
3672	return VINF_SUCCESS;
3673	}
3674
3675
3676	#if HC_ARCH_BITS == 32
3677	# ifdef VBOX_ENABLE_64_BITS_GUESTS
3678	/**
3679	* Check if guest state allows safe use of 32-bit switcher again.
3680	*
3681	* Segment bases and protected mode structures must be 32-bit addressable
3682	* because the 32-bit switcher will ignore high dword when writing these VMCS
3683	* fields. See @bugref{8432} for details.
3684	*
3685	* @returns true if safe, false if must continue to use the 64-bit switcher.
3686	* @param pCtx Pointer to the guest-CPU context.
3687	*
3688	* @remarks No-long-jump zone!!!
3689	*/
3690	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
3691	{
3692	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
3693	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
3694	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3695	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3696	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
3697	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3698	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
3699	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
3700	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3701	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3702
3703	/* All good, bases are 32-bit. */
3704	return true;
3705	}
3706	# endif /* VBOX_ENABLE_64_BITS_GUESTS */
3707
3708	# ifdef VBOX_STRICT
3709	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
3710	{
3711	switch (idxField)
3712	{
3713	case VMX_VMCS_GUEST_RIP:
3714	case VMX_VMCS_GUEST_RSP:
3715	case VMX_VMCS_GUEST_SYSENTER_EIP:
3716	case VMX_VMCS_GUEST_SYSENTER_ESP:
3717	case VMX_VMCS_GUEST_GDTR_BASE:
3718	case VMX_VMCS_GUEST_IDTR_BASE:
3719	case VMX_VMCS_GUEST_CS_BASE:
3720	case VMX_VMCS_GUEST_DS_BASE:
3721	case VMX_VMCS_GUEST_ES_BASE:
3722	case VMX_VMCS_GUEST_FS_BASE:
3723	case VMX_VMCS_GUEST_GS_BASE:
3724	case VMX_VMCS_GUEST_SS_BASE:
3725	case VMX_VMCS_GUEST_LDTR_BASE:
3726	case VMX_VMCS_GUEST_TR_BASE:
3727	case VMX_VMCS_GUEST_CR3:
3728	return true;
3729	}
3730	return false;
3731	}
3732
3733	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3734	{
3735	switch (idxField)
3736	{
3737	/* Read-only fields. */
3738	case VMX_VMCS_RO_EXIT_QUALIFICATION:
3739	return true;
3740	}
3741	/* Remaining readable fields should also be writable. */
3742	return hmR0VmxIsValidWriteField(idxField);
3743	}
3744	# endif /* VBOX_STRICT */
3745
3746
3747	/**
3748	* Executes the specified handler in 64-bit mode.
3749	*
3750	* @returns VBox status code (no informational status codes).
3751	* @param pVCpu The cross context virtual CPU structure.
3752	* @param enmOp The operation to perform.
3753	* @param cParams Number of parameters.
3754	* @param paParam Array of 32-bit parameters.
3755	*/
3756	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
3757	{
3758	PVM pVM = pVCpu->CTX_SUFF(pVM);
3759	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
3760	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
3761	Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
3762	Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
3763
3764	#ifdef VBOX_STRICT
3765	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
3766	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
3767
3768	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
3769	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
3770	#endif
3771
3772	/* Disable interrupts. */
3773	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
3774
3775	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
3776	RTCPUID idHostCpu = RTMpCpuId();
3777	CPUMR0SetLApic(pVCpu, idHostCpu);
3778	#endif
3779
3780	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
3781
3782	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3783	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3784
3785	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
3786	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3787	hmR0VmxClearVmcs(pVmcsInfo);
3788
3789	/* Leave VMX root mode and disable VMX. */
3790	VMXDisable();
3791	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3792
3793	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
3794	CPUMSetHyperEIP(pVCpu, enmOp);
3795	for (int i = (int)cParams - 1; i >= 0; i--)
3796	CPUMPushHyper(pVCpu, paParam[i]);
3797
3798	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
3799
3800	/* Call the switcher. */
3801	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
3802	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
3803
3804	/* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
3805	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
3806
3807	/* Re-enter VMX root mode. */
3808	int rc2 = VMXEnable(HCPhysCpuPage);
3809	if (RT_FAILURE(rc2))
3810	{
3811	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3812	ASMSetFlags(fOldEFlags);
3813	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
3814	return rc2;
3815	}
3816
3817	/* Restore the VMCS as the current VMCS. */
3818	rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
3819	AssertRC(rc2);
3820	Assert(!(ASMGetFlags() & X86_EFL_IF));
3821	ASMSetFlags(fOldEFlags);
3822	return rc;
3823	}
3824
3825
3826	/**
3827	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
3828	* supporting 64-bit guests.
3829	*
3830	* @returns VBox status code.
3831	* @param fResume Whether to VMLAUNCH or VMRESUME.
3832	* @param pCtx Pointer to the guest-CPU context.
3833	* @param pCache Pointer to the VMCS batch cache.
3834	* @param pVM The cross context VM structure.
3835	* @param pVCpu The cross context virtual CPU structure.
3836	*/
3837	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
3838	{
3839	NOREF(fResume);
3840
3841	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3842	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3843	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3844
3845	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3846	pCache->uPos = 1;
3847	pCache->interPD = PGMGetInterPaeCR3(pVM);
3848	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
3849	#endif
3850
3851	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3852	pCache->TestIn.HCPhysCpuPage = 0;
3853	pCache->TestIn.HCPhysVmcs = 0;
3854	pCache->TestIn.pCache = 0;
3855	pCache->TestOut.HCPhysVmcs = 0;
3856	pCache->TestOut.pCache = 0;
3857	pCache->TestOut.pCtx = 0;
3858	pCache->TestOut.eflags = 0;
3859	#else
3860	NOREF(pCache);
3861	#endif
3862
3863	uint32_t aParam[10];
3864	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
3865	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
3866	aParam[2] = RT_LO_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
3867	aParam[3] = RT_HI_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
3868	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
3869	aParam[5] = 0;
3870	aParam[6] = VM_RC_ADDR(pVM, pVM);
3871	aParam[7] = 0;
3872	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
3873	aParam[9] = 0;
3874
3875	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3876	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
3877	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
3878	#endif
3879	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
3880
3881	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3882	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
3883	Assert(pCtx->dr[4] == 10);
3884	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
3885	#endif
3886
3887	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3888	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
3889	AssertMsg(pCache->TestIn.HCPhysVmcs == pVmcsInfo->HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3890	pVmcsInfo->HCPhysVmcs));
3891	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3892	pCache->TestOut.HCPhysVmcs));
3893	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
3894	pCache->TestOut.pCache));
3895	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
3896	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
3897	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
3898	pCache->TestOut.pCtx));
3899	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
3900	#endif
3901	NOREF(pCtx);
3902	return rc;
3903	}
3904	#endif
3905
3906
3907	/**
3908	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
3909	* the VMCS.
3910	*
3911	* @returns VBox status code.
3912	*/
3913	static int hmR0VmxExportHostControlRegs(void)
3914	{
3915	RTCCUINTREG uReg = ASMGetCR0();
3916	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
3917	AssertRCReturn(rc, rc);
3918
3919	uReg = ASMGetCR3();
3920	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
3921	AssertRCReturn(rc, rc);
3922
3923	uReg = ASMGetCR4();
3924	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
3925	AssertRCReturn(rc, rc);
3926	return rc;
3927	}
3928
3929
3930	/**
3931	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
3932	* the host-state area in the VMCS.
3933	*
3934	* @returns VBox status code.
3935	* @param pVCpu The cross context virtual CPU structure.
3936	*/
3937	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
3938	{
3939	#if HC_ARCH_BITS == 64
3940	/**
3941	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
3942	* requirements. See hmR0VmxExportHostSegmentRegs().
3943	*/
3944	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
3945	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
3946	{ \
3947	bool fValidSelector = true; \
3948	if ((selValue) & X86_SEL_LDT) \
3949	{ \
3950	uint32_t uAttr = ASMGetSegAttr((selValue)); \
3951	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
3952	} \
3953	if (fValidSelector) \
3954	{ \
3955	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
3956	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
3957	} \
3958	(selValue) = 0; \
3959	}
3960
3961	/*
3962	* If we've executed guest code using hardware-assisted VMX, the host-state bits
3963	* will be messed up. We should -not- save the messed up state without restoring
3964	* the original host-state, see @bugref{7240}.
3965	*
3966	* This apparently can happen (most likely the FPU changes), deal with it rather than
3967	* asserting. Was observed booting Solaris 10u10 32-bit guest.
3968	*/
3969	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
3970	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
3971	{
3972	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
3973	pVCpu->idCpu));
3974	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
3975	}
3976	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
3977	#else
3978	RT_NOREF(pVCpu);
3979	#endif
3980
3981	/*
3982	* Host DS, ES, FS and GS segment registers.
3983	*/
3984	#if HC_ARCH_BITS == 64
3985	RTSEL uSelDS = ASMGetDS();
3986	RTSEL uSelES = ASMGetES();
3987	RTSEL uSelFS = ASMGetFS();
3988	RTSEL uSelGS = ASMGetGS();
3989	#else
3990	RTSEL uSelDS = 0;
3991	RTSEL uSelES = 0;
3992	RTSEL uSelFS = 0;
3993	RTSEL uSelGS = 0;
3994	#endif
3995
3996	/*
3997	* Host CS and SS segment registers.
3998	*/
3999	RTSEL uSelCS = ASMGetCS();
4000	RTSEL uSelSS = ASMGetSS();
4001
4002	/*
4003	* Host TR segment register.
4004	*/
4005	RTSEL uSelTR = ASMGetTR();
4006
4007	#if HC_ARCH_BITS == 64
4008	/*
4009	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
4010	* gain VM-entry and restore them before we get preempted.
4011	*
4012	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
4013	*/
4014	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
4015	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
4016	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
4017	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
4018	# undef VMXLOCAL_ADJUST_HOST_SEG
4019	#endif
4020
4021	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
4022	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
4023	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
4024	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
4025	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
4026	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
4027	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
4028	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
4029	Assert(uSelCS);
4030	Assert(uSelTR);
4031
4032	/* Write these host selector fields into the host-state area in the VMCS. */
4033	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
4034	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
4035	#if HC_ARCH_BITS == 64
4036	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
4037	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
4038	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
4039	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
4040	#else
4041	NOREF(uSelDS);
4042	NOREF(uSelES);
4043	NOREF(uSelFS);
4044	NOREF(uSelGS);
4045	#endif
4046	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
4047	AssertRCReturn(rc, rc);
4048
4049	/*
4050	* Host GDTR and IDTR.
4051	*/
4052	RTGDTR Gdtr;
4053	RTIDTR Idtr;
4054	RT_ZERO(Gdtr);
4055	RT_ZERO(Idtr);
4056	ASMGetGDTR(&Gdtr);
4057	ASMGetIDTR(&Idtr);
4058	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
4059	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
4060	AssertRCReturn(rc, rc);
4061
4062	#if HC_ARCH_BITS == 64
4063	/*
4064	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4065	* them to the maximum limit (0xffff) on every VM-exit.
4066	*/
4067	if (Gdtr.cbGdt != 0xffff)
4068	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4069
4070	/*
4071	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4072	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4073	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4074	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4075	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4076	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4077	* at 0xffff on hosts where we are sure it won't cause trouble.
4078	*/
4079	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4080	if (Idtr.cbIdt < 0x0fff)
4081	# else
4082	if (Idtr.cbIdt != 0xffff)
4083	# endif
4084	{
4085	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4086	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4087	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4088	}
4089	#endif
4090
4091	/*
4092	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4093	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4094	* RPL should be too in most cases.
4095	*/
4096	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4097	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4098
4099	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4100	#if HC_ARCH_BITS == 64
4101	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4102
4103	/*
4104	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4105	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4106	* restoration if the host has something else. Task switching is not supported in 64-bit
4107	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4108	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4109	*
4110	* [1] See Intel spec. 3.5 "System Descriptor Types".
4111	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4112	*/
4113	PVM pVM = pVCpu->CTX_SUFF(pVM);
4114	Assert(pDesc->System.u4Type == 11);
4115	if ( pDesc->System.u16LimitLow != 0x67
4116	\|\| pDesc->System.u4LimitHigh)
4117	{
4118	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4119	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4120	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4121	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4122	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4123	}
4124
4125	/*
4126	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4127	*/
4128	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4129	{
4130	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4131	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4132	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4133	{
4134	/* The GDT is read-only but the writable GDT is available. */
4135	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4136	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4137	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4138	AssertRCReturn(rc, rc);
4139	}
4140	}
4141	#else
4142	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4143	#endif
4144	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4145	AssertRCReturn(rc, rc);
4146
4147	/*
4148	* Host FS base and GS base.
4149	*/
4150	#if HC_ARCH_BITS == 64
4151	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4152	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4153	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4154	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4155	AssertRCReturn(rc, rc);
4156
4157	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4158	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4159	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4160	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4161	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4162	#endif
4163	return VINF_SUCCESS;
4164	}
4165
4166
4167	/**
4168	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4169	* host-state area of the VMCS.
4170	*
4171	* These MSRs will be automatically restored on the host after every successful
4172	* VM-exit.
4173	*
4174	* @returns VBox status code.
4175	* @param pVCpu The cross context virtual CPU structure.
4176	*
4177	* @remarks No-long-jump zone!!!
4178	*/
4179	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4180	{
4181	AssertPtr(pVCpu);
4182
4183	/*
4184	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4185	* rather than swapping them on every VM-entry.
4186	*/
4187	hmR0VmxLazySaveHostMsrs(pVCpu);
4188
4189	/*
4190	* Host Sysenter MSRs.
4191	*/
4192	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4193	#if HC_ARCH_BITS == 32
4194	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4195	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4196	#else
4197	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4198	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4199	#endif
4200	AssertRCReturn(rc, rc);
4201
4202	/*
4203	* Host EFER MSR.
4204	*
4205	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4206	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4207	*/
4208	PVM pVM = pVCpu->CTX_SUFF(pVM);
4209	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4210	{
4211	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4212	AssertRCReturn(rc, rc);
4213	}
4214
4215	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4216	* hmR0VmxExportGuestEntryExitCtls(). */
4217
4218	return VINF_SUCCESS;
4219	}
4220
4221
4222	/**
4223	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4224	*
4225	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4226	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4227	*
4228	* @returns true if we need to load guest EFER, false otherwise.
4229	* @param pVCpu The cross context virtual CPU structure.
4230	*
4231	* @remarks Requires EFER, CR4.
4232	* @remarks No-long-jump zone!!!
4233	*/
4234	static bool hmR0VmxShouldSwapEferMsr(PCVMCPU pVCpu)
4235	{
4236	#ifdef HMVMX_ALWAYS_SWAP_EFER
4237	RT_NOREF(pVCpu);
4238	return true;
4239	#else
4240	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4241	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4242	/* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4243	if (CPUMIsGuestInLongModeEx(pCtx))
4244	return false;
4245	#endif
4246
4247	PVM pVM = pVCpu->CTX_SUFF(pVM);
4248	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4249	uint64_t const u64GuestEfer = pCtx->msrEFER;
4250
4251	/*
4252	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4253	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4254	*/
4255	if ( CPUMIsGuestInLongModeEx(pCtx)
4256	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4257	return true;
4258
4259	/*
4260	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4261	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4262	*
4263	* See Intel spec. 4.5 "IA-32e Paging".
4264	* See Intel spec. 4.1.1 "Three Paging Modes".
4265	*
4266	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4267	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4268	*/
4269	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4270	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4271	if ( (pCtx->cr4 & X86_CR4_PAE)
4272	&& (pCtx->cr0 & X86_CR0_PG)
4273	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4274	{
4275	/* Assert that host is NX capable. */
4276	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4277	return true;
4278	}
4279
4280	return false;
4281	#endif
4282	}
4283
4284	/**
4285	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4286	* VMCS.
4287	*
4288	* This is typically required when the guest changes paging mode.
4289	*
4290	* @returns VBox status code.
4291	* @param pVCpu The cross context virtual CPU structure.
4292	* @param pVmxTransient The VMX-transient structure.
4293	*
4294	* @remarks Requires EFER.
4295	* @remarks No-long-jump zone!!!
4296	*/
4297	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4298	{
4299	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4300	{
4301	PVM pVM = pVCpu->CTX_SUFF(pVM);
4302	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4303
4304	/*
4305	* VM-entry controls.
4306	*/
4307	{
4308	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4309	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4310
4311	/*
4312	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4313	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4314	*
4315	* For nested-guests, this is a mandatory VM-entry control. It's also
4316	* required because we do not want to leak host bits to the nested-guest.
4317	*/
4318	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4319
4320	/*
4321	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4322	*
4323	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4324	* required to get the nested-guest working with hardware-assisted VMX execution.
4325	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested-hypervisor
4326	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4327	* here rather than while merging the guest VMCS controls.
4328	*/
4329	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4330	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4331	else
4332	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4333
4334	/*
4335	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4336	*
4337	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4338	* regardless of whether the nested-guest VMCS specifies it because we are free to
4339	* load whatever MSRs we require and we do not need to modify the guest visible copy
4340	* of the VM-entry MSR load area.
4341	*/
4342	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4343	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4344	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4345	else
4346	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4347
4348	/*
4349	* The following should -not- be set (since we're not in SMM mode):
4350	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4351	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4352	*/
4353
4354	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4355	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4356
4357	if ((fVal & fZap) == fVal)
4358	{ /* likely */ }
4359	else
4360	{
4361	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4362	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4363	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4364	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4365	}
4366
4367	/* Commit it to the VMCS. */
4368	if (pVmcsInfo->u32EntryCtls != fVal)
4369	{
4370	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4371	AssertRCReturn(rc, rc);
4372	pVmcsInfo->u32EntryCtls = fVal;
4373	}
4374	}
4375
4376	/*
4377	* VM-exit controls.
4378	*/
4379	{
4380	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4381	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4382
4383	/*
4384	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4385	* supported the 1-setting of this bit.
4386	*
4387	* For nested-guests, we set the "save debug controls" as the converse
4388	* "load debug controls" is mandatory for nested-guests anyway.
4389	*/
4390	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4391
4392	/*
4393	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4394	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4395	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4396	* hmR0VmxExportHostMsrs().
4397	*
4398	* For nested-guests, we always set this bit as we do not support 32-bit
4399	* hosts.
4400	*/
4401	#if HC_ARCH_BITS == 64
4402	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4403	#else
4404	Assert(!pVmxTransient->fIsNestedGuest);
4405	Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
4406	\|\| pVmcsInfo->pfnStartVM == VMXR0StartVM32);
4407	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
4408	if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
4409	{
4410	/* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
4411	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4412	}
4413	else
4414	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
4415	#endif
4416
4417	/*
4418	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4419	*
4420	* For nested-guests, we should use the "save IA32_EFER" control if we also
4421	* used the "load IA32_EFER" control while exporting VM-entry controls.
4422	*/
4423	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4424	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4425	{
4426	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4427	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4428	}
4429
4430	/*
4431	* Enable saving of the VMX-preemption timer value on VM-exit.
4432	* For nested-guests, currently not exposed/used.
4433	*/
4434	if ( pVM->hm.s.vmx.fUsePreemptTimer
4435	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4436	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4437
4438	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4439	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4440
4441	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4442	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4443	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4444
4445	if ((fVal & fZap) == fVal)
4446	{ /* likely */ }
4447	else
4448	{
4449	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4450	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4451	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4452	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4453	}
4454
4455	/* Commit it to the VMCS. */
4456	if (pVmcsInfo->u32ExitCtls != fVal)
4457	{
4458	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4459	AssertRCReturn(rc, rc);
4460	pVmcsInfo->u32ExitCtls = fVal;
4461	}
4462	}
4463
4464	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4465	}
4466	return VINF_SUCCESS;
4467	}
4468
4469
4470	/**
4471	* Sets the TPR threshold in the VMCS.
4472	*
4473	* @returns VBox status code.
4474	* @param pVCpu The cross context virtual CPU structure.
4475	* @param pVmcsInfo The VMCS info. object.
4476	* @param u32TprThreshold The TPR threshold (task-priority class only).
4477	*/
4478	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4479	{
4480	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4481	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4482	RT_NOREF2(pVCpu, pVmcsInfo);
4483	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4484	}
4485
4486
4487	/**
4488	* Exports the guest APIC TPR state into the VMCS.
4489	*
4490	* @returns VBox status code.
4491	* @param pVCpu The cross context virtual CPU structure.
4492	* @param pVmxTransient The VMX-transient structure.
4493	*
4494	* @remarks No-long-jump zone!!!
4495	*/
4496	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4497	{
4498	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4499	{
4500	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4501
4502	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4503	if (!pVmxTransient->fIsNestedGuest)
4504	{
4505	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4506	&& APICIsEnabled(pVCpu))
4507	{
4508	/*
4509	* Setup TPR shadowing.
4510	*/
4511	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4512	{
4513	bool fPendingIntr = false;
4514	uint8_t u8Tpr = 0;
4515	uint8_t u8PendingIntr = 0;
4516	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4517	AssertRCReturn(rc, rc);
4518
4519	/*
4520	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4521	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4522	* priority of the pending interrupt so we can deliver the interrupt. If there
4523	* are no interrupts pending, set threshold to 0 to not cause any
4524	* TPR-below-threshold VM-exits.
4525	*/
4526	Assert(pVmcsInfo->pbVirtApic);
4527	pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
4528	uint32_t u32TprThreshold = 0;
4529	if (fPendingIntr)
4530	{
4531	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4532	(which is the Task-Priority Class). */
4533	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4534	const uint8_t u8TprPriority = u8Tpr >> 4;
4535	if (u8PendingPriority <= u8TprPriority)
4536	u32TprThreshold = u8PendingPriority;
4537	}
4538
4539	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
4540	AssertRCReturn(rc, rc);
4541	}
4542	}
4543	}
4544	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4545	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4546	}
4547	return VINF_SUCCESS;
4548	}
4549
4550
4551	/**
4552	* Gets the guest interruptibility-state.
4553	*
4554	* @returns Guest's interruptibility-state.
4555	* @param pVCpu The cross context virtual CPU structure.
4556	* @param pVmcsInfo The VMCS info. object.
4557	*
4558	* @remarks No-long-jump zone!!!
4559	*/
4560	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
4561	{
4562	/*
4563	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4564	*/
4565	uint32_t fIntrState = 0;
4566	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4567	{
4568	/* If inhibition is active, RIP and RFLAGS should've been updated
4569	(i.e. read previously from the VMCS or from ring-3). */
4570	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4571	#ifdef VBOX_STRICT
4572	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
4573	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4574	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
4575	#endif
4576	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
4577	{
4578	if (pCtx->eflags.Bits.u1IF)
4579	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4580	else
4581	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
4582	}
4583	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4584	{
4585	/*
4586	* We can clear the inhibit force flag as even if we go back to the recompiler
4587	* without executing guest code in VT-x, the flag's condition to be cleared is
4588	* met and thus the cleared state is correct.
4589	*/
4590	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4591	}
4592	}
4593
4594	/*
4595	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
4596	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
4597	* setting this would block host-NMIs and IRET will not clear the blocking.
4598	*
4599	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
4600	*
4601	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
4602	*/
4603	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
4604	&& CPUMIsGuestNmiBlocking(pVCpu))
4605	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
4606
4607	return fIntrState;
4608	}
4609
4610
4611	/**
4612	* Exports the exception intercepts required for guest execution in the VMCS.
4613	*
4614	* @returns VBox status code.
4615	* @param pVCpu The cross context virtual CPU structure.
4616	* @param pVmxTransient The VMX-transient structure.
4617	*
4618	* @remarks No-long-jump zone!!!
4619	*/
4620	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4621	{
4622	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
4623	{
4624	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
4625	if ( !pVmxTransient->fIsNestedGuest
4626	&& pVCpu->hm.s.fGIMTrapXcptUD)
4627	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
4628	else
4629	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
4630
4631	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
4632	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
4633	}
4634	return VINF_SUCCESS;
4635	}
4636
4637
4638	/**
4639	* Exports the guest's RIP into the guest-state area in the VMCS.
4640	*
4641	* @returns VBox status code.
4642	* @param pVCpu The cross context virtual CPU structure.
4643	*
4644	* @remarks No-long-jump zone!!!
4645	*/
4646	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
4647	{
4648	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4649	{
4650	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
4651
4652	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
4653	AssertRCReturn(rc, rc);
4654
4655	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
4656	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
4657	}
4658	return VINF_SUCCESS;
4659	}
4660
4661
4662	/**
4663	* Exports the guest's RSP into the guest-state area in the VMCS.
4664	*
4665	* @returns VBox status code.
4666	* @param pVCpu The cross context virtual CPU structure.
4667	*
4668	* @remarks No-long-jump zone!!!
4669	*/
4670	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
4671	{
4672	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
4673	{
4674	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
4675
4676	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
4677	AssertRCReturn(rc, rc);
4678
4679	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
4680	}
4681	return VINF_SUCCESS;
4682	}
4683
4684
4685	/**
4686	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
4687	*
4688	* @returns VBox status code.
4689	* @param pVCpu The cross context virtual CPU structure.
4690	* @param pVmxTransient The VMX-transient structure.
4691	*
4692	* @remarks No-long-jump zone!!!
4693	*/
4694	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4695	{
4696	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
4697	{
4698	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
4699
4700	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
4701	Let us assert it as such and use 32-bit VMWRITE. */
4702	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
4703	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
4704	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
4705	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
4706
4707	/*
4708	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
4709	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
4710	* can run the real-mode guest code under Virtual 8086 mode.
4711	*/
4712	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4713	if (pVmcsInfo->RealMode.fRealOnV86Active)
4714	{
4715	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4716	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4717	Assert(!pVmxTransient->fIsNestedGuest);
4718	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
4719	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
4720	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
4721	}
4722
4723	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
4724	AssertRCReturn(rc, rc);
4725
4726	/*
4727	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
4728	*
4729	* We must avoid setting any automatic debug exceptions delivery when single-stepping
4730	* through the hypervisor debugger using EFLAGS.TF.
4731	*/
4732	if ( !pVmxTransient->fIsNestedGuest
4733	&& !pVCpu->hm.s.fSingleInstruction
4734	&& fEFlags.Bits.u1TF)
4735	{
4736	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
4737	* premature trips to ring-3 esp since IEM does not yet handle it. */
4738	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
4739	AssertRCReturn(rc, rc);
4740	}
4741	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
4742	* nested-guest VMCS. */
4743
4744	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
4745	Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
4746	}
4747	return VINF_SUCCESS;
4748	}
4749
4750
4751	/**
4752	* Exports the guest CR0 control register into the guest-state area in the VMCS.
4753	*
4754	* The guest FPU state is always pre-loaded hence we don't need to bother about
4755	* sharing FPU related CR0 bits between the guest and host.
4756	*
4757	* @returns VBox status code.
4758	* @param pVCpu The cross context virtual CPU structure.
4759	* @param pVmxTransient The VMX-transient structure.
4760	*
4761	* @remarks No-long-jump zone!!!
4762	*/
4763	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4764	{
4765	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
4766	{
4767	PVM pVM = pVCpu->CTX_SUFF(pVM);
4768	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4769
4770	/*
4771	* Figure out fixed CR0 bits in VMX operation.
4772	*/
4773	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4774	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4775	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4776	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
4777	else
4778	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
4779
4780	if (!pVmxTransient->fIsNestedGuest)
4781	{
4782	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4783	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4784	uint64_t const u64ShadowCr0 = u64GuestCr0;
4785	Assert(!RT_HI_U32(u64GuestCr0));
4786
4787	/*
4788	* Setup VT-x's view of the guest CR0.
4789	*/
4790	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
4791	if (pVM->hm.s.fNestedPaging)
4792	{
4793	if (CPUMIsGuestPagingEnabled(pVCpu))
4794	{
4795	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
4796	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
4797	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
4798	}
4799	else
4800	{
4801	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
4802	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
4803	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
4804	}
4805
4806	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
4807	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4808	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
4809	}
4810	else
4811	{
4812	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
4813	u64GuestCr0 \|= X86_CR0_WP;
4814	}
4815
4816	/*
4817	* Guest FPU bits.
4818	*
4819	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
4820	* using CR0.TS.
4821	*
4822	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
4823	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
4824	*/
4825	u64GuestCr0 \|= X86_CR0_NE;
4826
4827	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
4828	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
4829
4830	/*
4831	* Update exception intercepts.
4832	*/
4833	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
4834	if (pVmcsInfo->RealMode.fRealOnV86Active)
4835	{
4836	Assert(PDMVmmDevHeapIsEnabled(pVM));
4837	Assert(pVM->hm.s.vmx.pRealModeTSS);
4838	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
4839	}
4840	else
4841	{
4842	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
4843	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
4844	if (fInterceptMF)
4845	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
4846	}
4847
4848	/* Additional intercepts for debugging, define these yourself explicitly. */
4849	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4850	uXcptBitmap \|= 0
4851	\| RT_BIT(X86_XCPT_BP)
4852	\| RT_BIT(X86_XCPT_DE)
4853	\| RT_BIT(X86_XCPT_NM)
4854	\| RT_BIT(X86_XCPT_TS)
4855	\| RT_BIT(X86_XCPT_UD)
4856	\| RT_BIT(X86_XCPT_NP)
4857	\| RT_BIT(X86_XCPT_SS)
4858	\| RT_BIT(X86_XCPT_GP)
4859	\| RT_BIT(X86_XCPT_PF)
4860	\| RT_BIT(X86_XCPT_MF)
4861	;
4862	#elif defined(HMVMX_ALWAYS_TRAP_PF)
4863	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
4864	#endif
4865	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
4866	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
4867	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
4868
4869	/* Apply the fixed CR0 bits and enable caching. */
4870	u64GuestCr0 \|= fSetCr0;
4871	u64GuestCr0 &= fZapCr0;
4872	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4873
4874	/* Commit the CR0 and related fields to the guest VMCS. */
4875	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo Fix to 64-bit when we drop 32-bit. */
4876	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4877	if (uProcCtls != pVmcsInfo->u32ProcCtls)
4878	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4879	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
4880	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
4881	AssertRCReturn(rc, rc);
4882
4883	/* Update our caches. */
4884	pVmcsInfo->u32ProcCtls = uProcCtls;
4885	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
4886
4887	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4888	}
4889	else
4890	{
4891	/*
4892	* With nested-guests, we may have extended the guest/host mask here (since we
4893	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
4894	* mask can include more bits (to read from the nested-guest CR0 read-shadow) than
4895	* the guest hypervisor originally supplied. Thus, we should, in essence, copy
4896	* those bits from the nested-guest CR0 into the nested-guest CR0 read shadow.
4897	*/
4898	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4899	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4900	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(pVCpu, &pVCpu->cpum.GstCtx);
4901	Assert(!RT_HI_U32(u64GuestCr0));
4902	Assert(u64GuestCr0 & X86_CR0_NE);
4903
4904	/* Apply the fixed CR0 bits and enable caching. */
4905	u64GuestCr0 \|= fSetCr0;
4906	u64GuestCr0 &= fZapCr0;
4907	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4908
4909	/* Commit the CR0 and CR0 read shadow to the nested-guest VMCS. */
4910	/** @todo NSTVMX: Fix to 64-bit when we drop 32-bit. */
4911	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0);
4912	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4913	AssertRCReturn(rc, rc);
4914
4915	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4916	}
4917
4918	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
4919	}
4920
4921	return VINF_SUCCESS;
4922	}
4923
4924
4925	/**
4926	* Exports the guest control registers (CR3, CR4) into the guest-state area
4927	* in the VMCS.
4928	*
4929	* @returns VBox strict status code.
4930	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
4931	* without unrestricted guest access and the VMMDev is not presently
4932	* mapped (e.g. EFI32).
4933	*
4934	* @param pVCpu The cross context virtual CPU structure.
4935	* @param pVmxTransient The VMX-transient structure.
4936	*
4937	* @remarks No-long-jump zone!!!
4938	*/
4939	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4940	{
4941	int rc = VINF_SUCCESS;
4942	PVM pVM = pVCpu->CTX_SUFF(pVM);
4943
4944	/*
4945	* Guest CR2.
4946	* It's always loaded in the assembler code. Nothing to do here.
4947	*/
4948
4949	/*
4950	* Guest CR3.
4951	*/
4952	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
4953	{
4954	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
4955
4956	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
4957	if (pVM->hm.s.fNestedPaging)
4958	{
4959	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4960	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
4961
4962	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
4963	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
4964	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
4965	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
4966
4967	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
4968	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
4969	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
4970
4971	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
4972	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
4973	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
4974	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
4975	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
4976	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
4977	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
4978
4979	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
4980	AssertRCReturn(rc, rc);
4981
4982	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4983	if ( pVM->hm.s.vmx.fUnrestrictedGuest
4984	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
4985	{
4986	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
4987	if (CPUMIsGuestInPAEModeEx(pCtx))
4988	{
4989	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
4990	AssertRCReturn(rc, rc);
4991	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
4992	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
4993	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
4994	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
4995	AssertRCReturn(rc, rc);
4996	}
4997
4998	/*
4999	* The guest's view of its CR3 is unblemished with nested paging when the
5000	* guest is using paging or we have unrestricted guest execution to handle
5001	* the guest when it's not using paging.
5002	*/
5003	GCPhysGuestCR3 = pCtx->cr3;
5004	}
5005	else
5006	{
5007	/*
5008	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
5009	* thinks it accesses physical memory directly, we use our identity-mapped
5010	* page table to map guest-linear to guest-physical addresses. EPT takes care
5011	* of translating it to host-physical addresses.
5012	*/
5013	RTGCPHYS GCPhys;
5014	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
5015
5016	/* We obtain it here every time as the guest could have relocated this PCI region. */
5017	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
5018	if (RT_SUCCESS(rc))
5019	{ /* likely */ }
5020	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
5021	{
5022	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
5023	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
5024	}
5025	else
5026	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
5027
5028	GCPhysGuestCR3 = GCPhys;
5029	}
5030
5031	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
5032	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
5033	AssertRCReturn(rc, rc);
5034	}
5035	else
5036	{
5037	/* Non-nested paging case, just use the hypervisor's CR3. */
5038	RTHCPHYS const HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
5039
5040	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
5041	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
5042	AssertRCReturn(rc, rc);
5043	}
5044
5045	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
5046	}
5047
5048	/*
5049	* Guest CR4.
5050	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
5051	*/
5052	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
5053	{
5054	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5055	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5056
5057	/*
5058	* Figure out fixed CR4 bits in VMX operation.
5059	*/
5060	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5061	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5062
5063	/*
5064	* With nested-guests, we may have extended the guest/host mask here (since we
5065	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
5066	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
5067	* the guest hypervisor originally supplied. Thus, we should, in essence, copy
5068	* those bits from the nested-guest CR4 into the nested-guest CR4 read shadow.
5069	*/
5070	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5071	uint64_t u64GuestCr4 = pCtx->cr4;
5072	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
5073	? pCtx->cr4
5074	: CPUMGetGuestVmxMaskedCr4(pVCpu, pCtx);
5075	Assert(!RT_HI_U32(u64GuestCr4));
5076
5077	/*
5078	* Setup VT-x's view of the guest CR4.
5079	*
5080	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5081	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5082	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5083	*
5084	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5085	*/
5086	if (pVmcsInfo->RealMode.fRealOnV86Active)
5087	{
5088	Assert(pVM->hm.s.vmx.pRealModeTSS);
5089	Assert(PDMVmmDevHeapIsEnabled(pVM));
5090	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5091	}
5092
5093	if (pVM->hm.s.fNestedPaging)
5094	{
5095	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5096	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5097	{
5098	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5099	u64GuestCr4 \|= X86_CR4_PSE;
5100	/* Our identity mapping is a 32-bit page directory. */
5101	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5102	}
5103	/* else use guest CR4.*/
5104	}
5105	else
5106	{
5107	Assert(!pVmxTransient->fIsNestedGuest);
5108
5109	/*
5110	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5111	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5112	*/
5113	switch (pVCpu->hm.s.enmShadowMode)
5114	{
5115	case PGMMODE_REAL: /* Real-mode. */
5116	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5117	case PGMMODE_32_BIT: /* 32-bit paging. */
5118	{
5119	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5120	break;
5121	}
5122
5123	case PGMMODE_PAE: /* PAE paging. */
5124	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5125	{
5126	u64GuestCr4 \|= X86_CR4_PAE;
5127	break;
5128	}
5129
5130	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5131	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5132	#ifdef VBOX_ENABLE_64_BITS_GUESTS
5133	break;
5134	#endif
5135	default:
5136	AssertFailed();
5137	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5138	}
5139	}
5140
5141	/* Apply the fixed CR4 bits (mainly CR4.VMXE). */
5142	u64GuestCr4 \|= fSetCr4;
5143	u64GuestCr4 &= fZapCr4;
5144
5145	/* Commit the CR4 and CR4 read shadow to the guest VMCS. */
5146	/** @todo Fix to 64-bit when we drop 32-bit. */
5147	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u64GuestCr4);
5148	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
5149	AssertRCReturn(rc, rc);
5150
5151	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5152	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5153
5154	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5155
5156	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5157	}
5158	return rc;
5159	}
5160
5161
5162	/**
5163	* Exports the guest debug registers into the guest-state area in the VMCS.
5164	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5165	*
5166	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5167	*
5168	* @returns VBox status code.
5169	* @param pVCpu The cross context virtual CPU structure.
5170	* @param pVmxTransient The VMX-transient structure.
5171	*
5172	* @remarks No-long-jump zone!!!
5173	*/
5174	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5175	{
5176	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5177
5178	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5179	* stepping. */
5180	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5181	if (pVmxTransient->fIsNestedGuest)
5182	{
5183	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5184	AssertRCReturn(rc, rc);
5185	return VINF_SUCCESS;
5186	}
5187
5188	#ifdef VBOX_STRICT
5189	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5190	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5191	{
5192	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5193	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5194	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5195	}
5196	#endif
5197
5198	bool fSteppingDB = false;
5199	bool fInterceptMovDRx = false;
5200	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5201	if (pVCpu->hm.s.fSingleInstruction)
5202	{
5203	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5204	PVM pVM = pVCpu->CTX_SUFF(pVM);
5205	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5206	{
5207	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5208	Assert(fSteppingDB == false);
5209	}
5210	else
5211	{
5212	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5213	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5214	pVCpu->hm.s.fClearTrapFlag = true;
5215	fSteppingDB = true;
5216	}
5217	}
5218
5219	uint32_t u32GuestDr7;
5220	if ( fSteppingDB
5221	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5222	{
5223	/*
5224	* Use the combined guest and host DRx values found in the hypervisor register set
5225	* because the hypervisor debugger has breakpoints active or someone is single stepping
5226	* on the host side without a monitor trap flag.
5227	*
5228	* Note! DBGF expects a clean DR6 state before executing guest code.
5229	*/
5230	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5231	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5232	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
5233	{
5234	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5235	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
5236	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
5237	}
5238	else
5239	#endif
5240	if (!CPUMIsHyperDebugStateActive(pVCpu))
5241	{
5242	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5243	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5244	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5245	}
5246
5247	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5248	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
5249	pVCpu->hm.s.fUsingHyperDR7 = true;
5250	fInterceptMovDRx = true;
5251	}
5252	else
5253	{
5254	/*
5255	* If the guest has enabled debug registers, we need to load them prior to
5256	* executing guest code so they'll trigger at the right time.
5257	*/
5258	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5259	{
5260	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5261	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5262	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
5263	{
5264	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5265	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
5266	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
5267	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5268	}
5269	else
5270	#endif
5271	if (!CPUMIsGuestDebugStateActive(pVCpu))
5272	{
5273	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5274	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5275	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5276	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5277	}
5278	Assert(!fInterceptMovDRx);
5279	}
5280	/*
5281	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5282	* must intercept #DB in order to maintain a correct DR6 guest value, and
5283	* because we need to intercept it to prevent nested #DBs from hanging the
5284	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5285	*/
5286	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5287	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
5288	&& !CPUMIsGuestDebugStateActive(pVCpu))
5289	#else
5290	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5291	#endif
5292	{
5293	fInterceptMovDRx = true;
5294	}
5295
5296	/* Update DR7 with the actual guest value. */
5297	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5298	pVCpu->hm.s.fUsingHyperDR7 = false;
5299	}
5300
5301	if (fInterceptMovDRx)
5302	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5303	else
5304	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5305
5306	/*
5307	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5308	* monitor-trap flag and update our cache.
5309	*/
5310	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5311	{
5312	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5313	AssertRCReturn(rc2, rc2);
5314	pVmcsInfo->u32ProcCtls = uProcCtls;
5315	}
5316
5317	/*
5318	* Update guest DR7.
5319	*/
5320	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
5321	AssertRCReturn(rc, rc);
5322
5323	/*
5324	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
5325	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
5326	*
5327	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5328	*/
5329	if (fSteppingDB)
5330	{
5331	Assert(pVCpu->hm.s.fSingleInstruction);
5332	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
5333
5334	uint32_t fIntrState = 0;
5335	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
5336	AssertRCReturn(rc, rc);
5337
5338	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5339	{
5340	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5341	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5342	AssertRCReturn(rc, rc);
5343	}
5344	}
5345
5346	return VINF_SUCCESS;
5347	}
5348
5349
5350	#ifdef VBOX_STRICT
5351	/**
5352	* Strict function to validate segment registers.
5353	*
5354	* @param pVCpu The cross context virtual CPU structure.
5355	* @param pVmcsInfo The VMCS info. object.
5356	*
5357	* @remarks Will import guest CR0 on strict builds during validation of
5358	* segments.
5359	*/
5360	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
5361	{
5362	/*
5363	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5364	*
5365	* The reason we check for attribute value 0 in this function and not just the unusable bit is
5366	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
5367	* unusable bit and doesn't change the guest-context value.
5368	*/
5369	PVM pVM = pVCpu->CTX_SUFF(pVM);
5370	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5371	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
5372	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
5373	&& ( !CPUMIsGuestInRealModeEx(pCtx)
5374	&& !CPUMIsGuestInV86ModeEx(pCtx)))
5375	{
5376	/* Protected mode checks */
5377	/* CS */
5378	Assert(pCtx->cs.Attr.n.u1Present);
5379	Assert(!(pCtx->cs.Attr.u & 0xf00));
5380	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
5381	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5382	\|\| !(pCtx->cs.Attr.n.u1Granularity));
5383	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
5384	\|\| (pCtx->cs.Attr.n.u1Granularity));
5385	/* CS cannot be loaded with NULL in protected mode. */
5386	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
5387	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5388	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
5389	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5390	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
5391	else
5392	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
5393	/* SS */
5394	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5395	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
5396	if ( !(pCtx->cr0 & X86_CR0_PE)
5397	\|\| pCtx->cs.Attr.n.u4Type == 3)
5398	{
5399	Assert(!pCtx->ss.Attr.n.u2Dpl);
5400	}
5401	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
5402	{
5403	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5404	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
5405	Assert(pCtx->ss.Attr.n.u1Present);
5406	Assert(!(pCtx->ss.Attr.u & 0xf00));
5407	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
5408	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
5409	\|\| !(pCtx->ss.Attr.n.u1Granularity));
5410	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
5411	\|\| (pCtx->ss.Attr.n.u1Granularity));
5412	}
5413	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
5414	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
5415	{
5416	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5417	Assert(pCtx->ds.Attr.n.u1Present);
5418	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
5419	Assert(!(pCtx->ds.Attr.u & 0xf00));
5420	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
5421	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
5422	\|\| !(pCtx->ds.Attr.n.u1Granularity));
5423	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
5424	\|\| (pCtx->ds.Attr.n.u1Granularity));
5425	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5426	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
5427	}
5428	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
5429	{
5430	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5431	Assert(pCtx->es.Attr.n.u1Present);
5432	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
5433	Assert(!(pCtx->es.Attr.u & 0xf00));
5434	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
5435	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
5436	\|\| !(pCtx->es.Attr.n.u1Granularity));
5437	Assert( !(pCtx->es.u32Limit & 0xfff00000)
5438	\|\| (pCtx->es.Attr.n.u1Granularity));
5439	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5440	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
5441	}
5442	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
5443	{
5444	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5445	Assert(pCtx->fs.Attr.n.u1Present);
5446	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
5447	Assert(!(pCtx->fs.Attr.u & 0xf00));
5448	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
5449	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
5450	\|\| !(pCtx->fs.Attr.n.u1Granularity));
5451	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
5452	\|\| (pCtx->fs.Attr.n.u1Granularity));
5453	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5454	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5455	}
5456	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
5457	{
5458	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5459	Assert(pCtx->gs.Attr.n.u1Present);
5460	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
5461	Assert(!(pCtx->gs.Attr.u & 0xf00));
5462	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
5463	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
5464	\|\| !(pCtx->gs.Attr.n.u1Granularity));
5465	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
5466	\|\| (pCtx->gs.Attr.n.u1Granularity));
5467	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5468	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5469	}
5470	/* 64-bit capable CPUs. */
5471	# if HC_ARCH_BITS == 64
5472	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5473	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
5474	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
5475	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
5476	# endif
5477	}
5478	else if ( CPUMIsGuestInV86ModeEx(pCtx)
5479	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
5480	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
5481	{
5482	/* Real and v86 mode checks. */
5483	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
5484	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
5485	if (pVmcsInfo->RealMode.fRealOnV86Active)
5486	{
5487	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
5488	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
5489	}
5490	else
5491	{
5492	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
5493	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
5494	}
5495
5496	/* CS */
5497	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
5498	Assert(pCtx->cs.u32Limit == 0xffff);
5499	Assert(u32CSAttr == 0xf3);
5500	/* SS */
5501	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
5502	Assert(pCtx->ss.u32Limit == 0xffff);
5503	Assert(u32SSAttr == 0xf3);
5504	/* DS */
5505	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
5506	Assert(pCtx->ds.u32Limit == 0xffff);
5507	Assert(u32DSAttr == 0xf3);
5508	/* ES */
5509	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
5510	Assert(pCtx->es.u32Limit == 0xffff);
5511	Assert(u32ESAttr == 0xf3);
5512	/* FS */
5513	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
5514	Assert(pCtx->fs.u32Limit == 0xffff);
5515	Assert(u32FSAttr == 0xf3);
5516	/* GS */
5517	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
5518	Assert(pCtx->gs.u32Limit == 0xffff);
5519	Assert(u32GSAttr == 0xf3);
5520	/* 64-bit capable CPUs. */
5521	# if HC_ARCH_BITS == 64
5522	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5523	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
5524	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
5525	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
5526	# endif
5527	}
5528	}
5529	#endif /* VBOX_STRICT */
5530
5531
5532	/**
5533	* Exports a guest segment register into the guest-state area in the VMCS.
5534	*
5535	* @returns VBox status code.
5536	* @param pVCpu The cross context virtual CPU structure.
5537	* @param pVmcsInfo The VMCS info. object.
5538	* @param iSegReg The segment register number (X86_SREG_XXX).
5539	* @param pSelReg Pointer to the segment selector.
5540	*
5541	* @remarks No-long-jump zone!!!
5542	*/
5543	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
5544	{
5545	Assert(iSegReg < X86_SREG_COUNT);
5546	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
5547	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
5548	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
5549	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
5550
5551	uint32_t u32Access = pSelReg->Attr.u;
5552	if (pVmcsInfo->RealMode.fRealOnV86Active)
5553	{
5554	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
5555	u32Access = 0xf3;
5556	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5557	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5558	RT_NOREF_PV(pVCpu);
5559	}
5560	else
5561	{
5562	/*
5563	* The way to differentiate between whether this is really a null selector or was just
5564	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
5565	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
5566	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
5567	* NULL selectors loaded in protected-mode have their attribute as 0.
5568	*/
5569	if (!u32Access)
5570	u32Access = X86DESCATTR_UNUSABLE;
5571	}
5572
5573	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
5574	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
5575	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
5576
5577	/*
5578	* Commit it to the VMCS.
5579	*/
5580	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
5581	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
5582	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
5583	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
5584	AssertRCReturn(rc, rc);
5585	return rc;
5586	}
5587
5588
5589	/**
5590	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
5591	* area in the VMCS.
5592	*
5593	* @returns VBox status code.
5594	* @param pVCpu The cross context virtual CPU structure.
5595	* @param pVmxTransient The VMX-transient structure.
5596	*
5597	* @remarks Will import guest CR0 on strict builds during validation of
5598	* segments.
5599	* @remarks No-long-jump zone!!!
5600	*/
5601	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5602	{
5603	int rc = VERR_INTERNAL_ERROR_5;
5604	PVM pVM = pVCpu->CTX_SUFF(pVM);
5605	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5606	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5607
5608	/*
5609	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
5610	*/
5611	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
5612	{
5613	#ifdef VBOX_WITH_REM
5614	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
5615	{
5616	Assert(!pVmxTransient->fIsNestedGuest);
5617	Assert(pVM->hm.s.vmx.pRealModeTSS);
5618	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
5619	if ( pVmcsInfo->fWasInRealMode
5620	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
5621	{
5622	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
5623	in real-mode (e.g. OpenBSD 4.0) */
5624	REMFlushTBs(pVM);
5625	Log4Func(("Switch to protected mode detected!\n"));
5626	pVmcsInfo->fWasInRealMode = false;
5627	}
5628	}
5629	#endif
5630	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
5631	{
5632	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
5633	if (pVmcsInfo->RealMode.fRealOnV86Active)
5634	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
5635	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
5636	AssertRCReturn(rc, rc);
5637	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
5638	}
5639
5640	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
5641	{
5642	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
5643	if (pVmcsInfo->RealMode.fRealOnV86Active)
5644	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
5645	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
5646	AssertRCReturn(rc, rc);
5647	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
5648	}
5649
5650	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
5651	{
5652	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
5653	if (pVmcsInfo->RealMode.fRealOnV86Active)
5654	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
5655	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
5656	AssertRCReturn(rc, rc);
5657	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
5658	}
5659
5660	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
5661	{
5662	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
5663	if (pVmcsInfo->RealMode.fRealOnV86Active)
5664	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
5665	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
5666	AssertRCReturn(rc, rc);
5667	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
5668	}
5669
5670	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
5671	{
5672	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
5673	if (pVmcsInfo->RealMode.fRealOnV86Active)
5674	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
5675	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
5676	AssertRCReturn(rc, rc);
5677	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
5678	}
5679
5680	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
5681	{
5682	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
5683	if (pVmcsInfo->RealMode.fRealOnV86Active)
5684	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
5685	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
5686	AssertRCReturn(rc, rc);
5687	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
5688	}
5689
5690	#ifdef VBOX_STRICT
5691	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
5692	#endif
5693	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
5694	pCtx->cs.Attr.u));
5695	}
5696
5697	/*
5698	* Guest TR.
5699	*/
5700	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
5701	{
5702	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
5703
5704	/*
5705	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
5706	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
5707	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
5708	*/
5709	uint16_t u16Sel;
5710	uint32_t u32Limit;
5711	uint64_t u64Base;
5712	uint32_t u32AccessRights;
5713	if (!pVmcsInfo->RealMode.fRealOnV86Active)
5714	{
5715	u16Sel = pCtx->tr.Sel;
5716	u32Limit = pCtx->tr.u32Limit;
5717	u64Base = pCtx->tr.u64Base;
5718	u32AccessRights = pCtx->tr.Attr.u;
5719	}
5720	else
5721	{
5722	Assert(!pVmxTransient->fIsNestedGuest);
5723	Assert(pVM->hm.s.vmx.pRealModeTSS);
5724	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
5725
5726	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
5727	RTGCPHYS GCPhys;
5728	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
5729	AssertRCReturn(rc, rc);
5730
5731	X86DESCATTR DescAttr;
5732	DescAttr.u = 0;
5733	DescAttr.n.u1Present = 1;
5734	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
5735
5736	u16Sel = 0;
5737	u32Limit = HM_VTX_TSS_SIZE;
5738	u64Base = GCPhys;
5739	u32AccessRights = DescAttr.u;
5740	}
5741
5742	/* Validate. */
5743	Assert(!(u16Sel & RT_BIT(2)));
5744	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
5745	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
5746	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
5747	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
5748	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
5749	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
5750	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
5751	Assert( (u32Limit & 0xfff) == 0xfff
5752	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
5753	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
5754	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
5755
5756	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
5757	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
5758	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
5759	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
5760	AssertRCReturn(rc, rc);
5761
5762	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
5763	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
5764	}
5765
5766	/*
5767	* Guest GDTR.
5768	*/
5769	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
5770	{
5771	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
5772
5773	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
5774	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
5775	AssertRCReturn(rc, rc);
5776
5777	/* Validate. */
5778	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5779
5780	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
5781	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
5782	}
5783
5784	/*
5785	* Guest LDTR.
5786	*/
5787	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
5788	{
5789	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
5790
5791	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
5792	uint32_t u32Access;
5793	if ( !pVmxTransient->fIsNestedGuest
5794	&& !pCtx->ldtr.Attr.u)
5795	u32Access = X86DESCATTR_UNUSABLE;
5796	else
5797	u32Access = pCtx->ldtr.Attr.u;
5798
5799	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
5800	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
5801	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
5802	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
5803	AssertRCReturn(rc, rc);
5804
5805	/* Validate. */
5806	if (!(u32Access & X86DESCATTR_UNUSABLE))
5807	{
5808	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
5809	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
5810	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
5811	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
5812	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
5813	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
5814	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
5815	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
5816	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
5817	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
5818	}
5819
5820	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
5821	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
5822	}
5823
5824	/*
5825	* Guest IDTR.
5826	*/
5827	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
5828	{
5829	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
5830
5831	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
5832	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
5833	AssertRCReturn(rc, rc);
5834
5835	/* Validate. */
5836	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5837
5838	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
5839	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
5840	}
5841
5842	return VINF_SUCCESS;
5843	}
5844
5845
5846	/**
5847	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
5848	* areas.
5849	*
5850	* These MSRs will automatically be loaded to the host CPU on every successful
5851	* VM-entry and stored from the host CPU on every successful VM-exit.
5852	*
5853	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
5854	* actual host MSR values are not- updated here for performance reasons. See
5855	* hmR0VmxExportHostMsrs().
5856	*
5857	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
5858	*
5859	* @returns VBox status code.
5860	* @param pVCpu The cross context virtual CPU structure.
5861	* @param pVmxTransient The VMX-transient structure.
5862	*
5863	* @remarks No-long-jump zone!!!
5864	*/
5865	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5866	{
5867	AssertPtr(pVCpu);
5868	AssertPtr(pVmxTransient);
5869
5870	PVM pVM = pVCpu->CTX_SUFF(pVM);
5871	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5872
5873	/*
5874	* MSRs that we use the auto-load/store MSR area in the VMCS.
5875	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
5876	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
5877	*
5878	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
5879	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
5880	* emulation, nothing to do here.
5881	*/
5882	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
5883	{
5884	if ( !pVmxTransient->fIsNestedGuest
5885	&& pVM->hm.s.fAllow64BitGuests)
5886	{
5887	#if HC_ARCH_BITS == 32
5888	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
5889	Assert(!pVmxTransient->fIsNestedGuest);
5890
5891	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
5892	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
5893	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
5894	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
5895	AssertRCReturn(rc, rc);
5896	#endif
5897	}
5898	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
5899	}
5900
5901	/*
5902	* Guest Sysenter MSRs.
5903	*/
5904	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
5905	{
5906	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
5907
5908	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
5909	{
5910	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
5911	AssertRCReturn(rc, rc);
5912	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
5913	}
5914
5915	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
5916	{
5917	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
5918	AssertRCReturn(rc, rc);
5919	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
5920	}
5921
5922	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
5923	{
5924	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
5925	AssertRCReturn(rc, rc);
5926	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
5927	}
5928	}
5929
5930	/*
5931	* Guest/host EFER MSR.
5932	*/
5933	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
5934	{
5935	/* Whether we are using the VMCS to swap the EFER MSR must have been
5936	determined earlier while exporting VM-entry/VM-exit controls. */
5937	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
5938	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
5939
5940	if (hmR0VmxShouldSwapEferMsr(pVCpu))
5941	{
5942	/*
5943	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
5944	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
5945	*/
5946	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
5947	{
5948	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
5949	AssertRCReturn(rc, rc);
5950	}
5951	else
5952	{
5953	/*
5954	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
5955	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
5956	*/
5957	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
5958	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5959	AssertRCReturn(rc, rc);
5960	}
5961	}
5962	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
5963	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
5964
5965	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
5966	}
5967
5968	/*
5969	* Other MSRs.
5970	* Speculation Control (R/W).
5971	*/
5972	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
5973	{
5974	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
5975	if (pVM->cpum.ro.GuestFeatures.fIbrs)
5976	{
5977	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
5978	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5979	AssertRCReturn(rc, rc);
5980	}
5981	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
5982	}
5983
5984	return VINF_SUCCESS;
5985	}
5986
5987
5988	/**
5989	* Selects up the appropriate function to run guest code.
5990	*
5991	* @returns VBox status code.
5992	* @param pVCpu The cross context virtual CPU structure.
5993	* @param pVmxTransient The VMX-transient structure.
5994	*
5995	* @remarks No-long-jump zone!!!
5996	*/
5997	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5998	{
5999	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6000	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6001
6002	if (CPUMIsGuestInLongModeEx(pCtx))
6003	{
6004	#ifndef VBOX_ENABLE_64_BITS_GUESTS
6005	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
6006	#endif
6007	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
6008	#if HC_ARCH_BITS == 32
6009	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
6010	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
6011	{
6012	#ifdef VBOX_STRICT
6013	if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
6014	{
6015	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
6016	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
6017	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
6018	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
6019	("fCtxChanged=%#RX64\n", fCtxChanged));
6020	}
6021	#endif
6022	pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
6023
6024	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
6025	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
6026	pVmcsInfo->fSwitchedTo64on32 = true;
6027	Log4Func(("Selected 64-bit switcher\n"));
6028	}
6029	#else
6030	/* 64-bit host. */
6031	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
6032	#endif
6033	}
6034	else
6035	{
6036	/* Guest is not in long mode, use the 32-bit handler. */
6037	#if HC_ARCH_BITS == 32
6038	if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
6039	&& !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
6040	&& pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
6041	{
6042	# ifdef VBOX_STRICT
6043	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
6044	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
6045	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
6046	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
6047	("fCtxChanged=%#RX64\n", fCtxChanged));
6048	# endif
6049	}
6050	# ifdef VBOX_ENABLE_64_BITS_GUESTS
6051	/*
6052	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
6053	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
6054	* switcher flag now because we know the guest is in a sane state where it's safe
6055	* to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
6056	* the much faster 32-bit switcher again.
6057	*/
6058	if (!pVmcsInfo->fSwitchedTo64on32)
6059	{
6060	if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
6061	Log4Func(("Selected 32-bit switcher\n"));
6062	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6063	}
6064	else
6065	{
6066	Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
6067	if ( pVmcsInfo->RealMode.fRealOnV86Active
6068	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
6069	{
6070	pVmcsInfo->fSwitchedTo64on32 = false;
6071	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6072	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
6073	\| HM_CHANGED_VMX_ENTRY_EXIT_CTLS
6074	\| HM_CHANGED_HOST_CONTEXT);
6075	Log4Func(("Selected 32-bit switcher (safe)\n"));
6076	}
6077	}
6078	# else
6079	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6080	# endif
6081	#else
6082	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6083	#endif
6084	}
6085	Assert(pVmcsInfo->pfnStartVM);
6086	return VINF_SUCCESS;
6087	}
6088
6089
6090	/**
6091	* Wrapper for running the guest code in VT-x.
6092	*
6093	* @returns VBox status code, no informational status codes.
6094	* @param pVCpu The cross context virtual CPU structure.
6095	* @param pVmxTransient The VMX-transient structure.
6096	*
6097	* @remarks No-long-jump zone!!!
6098	*/
6099	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6100	{
6101	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6102	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6103	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6104
6105	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6106
6107	/*
6108	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6109	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6110	* callee-saved and thus the need for this XMM wrapper.
6111	*
6112	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6113	*/
6114	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6115	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6116	PVM pVM = pVCpu->CTX_SUFF(pVM);
6117	#ifdef VBOX_WITH_KERNEL_USING_XMM
6118	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6119	#else
6120	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
6121	#endif
6122	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6123	return rc;
6124	}
6125
6126
6127	/**
6128	* Reports world-switch error and dumps some useful debug info.
6129	*
6130	* @param pVCpu The cross context virtual CPU structure.
6131	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6132	* @param pVmxTransient The VMX-transient structure (only
6133	* exitReason updated).
6134	*/
6135	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6136	{
6137	Assert(pVCpu);
6138	Assert(pVmxTransient);
6139	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6140
6141	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6142	switch (rcVMRun)
6143	{
6144	case VERR_VMX_INVALID_VMXON_PTR:
6145	AssertFailed();
6146	break;
6147	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6148	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6149	{
6150	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6151	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6152	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
6153	AssertRC(rc);
6154
6155	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6156	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6157	Cannot do it here as we may have been long preempted. */
6158
6159	#ifdef VBOX_STRICT
6160	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6161	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6162	pVmxTransient->uExitReason));
6163	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6164	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6165	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6166	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6167	else
6168	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6169	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6170	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6171
6172	/* VMX control bits. */
6173	uint32_t u32Val;
6174	uint64_t u64Val;
6175	RTHCUINTREG uHCReg;
6176	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
6177	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
6178	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
6179	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
6180	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
6181	{
6182	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
6183	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
6184	}
6185	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
6186	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
6187	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
6188	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
6189	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
6190	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
6191	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
6192	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
6193	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
6194	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
6195	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
6196	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
6197	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
6198	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
6199	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
6200	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
6201	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6202	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
6203	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6204	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
6205	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
6206	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
6207	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
6208	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
6209	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
6210	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
6211	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
6212	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
6213	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
6214	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6215	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
6216	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
6217	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
6218	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6219	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6220	{
6221	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
6222	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
6223	}
6224
6225	/* Guest bits. */
6226	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
6227	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
6228	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
6229	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
6230	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
6231	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
6232	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
6233	{
6234	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
6235	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
6236	}
6237
6238	/* Host bits. */
6239	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
6240	Log4(("Host CR0 %#RHr\n", uHCReg));
6241	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
6242	Log4(("Host CR3 %#RHr\n", uHCReg));
6243	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
6244	Log4(("Host CR4 %#RHr\n", uHCReg));
6245
6246	RTGDTR HostGdtr;
6247	PCX86DESCHC pDesc;
6248	ASMGetGDTR(&HostGdtr);
6249	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
6250	Log4(("Host CS %#08x\n", u32Val));
6251	if (u32Val < HostGdtr.cbGdt)
6252	{
6253	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6254	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
6255	}
6256
6257	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
6258	Log4(("Host DS %#08x\n", u32Val));
6259	if (u32Val < HostGdtr.cbGdt)
6260	{
6261	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6262	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
6263	}
6264
6265	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
6266	Log4(("Host ES %#08x\n", u32Val));
6267	if (u32Val < HostGdtr.cbGdt)
6268	{
6269	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6270	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
6271	}
6272
6273	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
6274	Log4(("Host FS %#08x\n", u32Val));
6275	if (u32Val < HostGdtr.cbGdt)
6276	{
6277	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6278	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
6279	}
6280
6281	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
6282	Log4(("Host GS %#08x\n", u32Val));
6283	if (u32Val < HostGdtr.cbGdt)
6284	{
6285	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6286	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
6287	}
6288
6289	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
6290	Log4(("Host SS %#08x\n", u32Val));
6291	if (u32Val < HostGdtr.cbGdt)
6292	{
6293	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6294	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
6295	}
6296
6297	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
6298	Log4(("Host TR %#08x\n", u32Val));
6299	if (u32Val < HostGdtr.cbGdt)
6300	{
6301	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6302	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
6303	}
6304
6305	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
6306	Log4(("Host TR Base %#RHv\n", uHCReg));
6307	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
6308	Log4(("Host GDTR Base %#RHv\n", uHCReg));
6309	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
6310	Log4(("Host IDTR Base %#RHv\n", uHCReg));
6311	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
6312	Log4(("Host SYSENTER CS %#08x\n", u32Val));
6313	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
6314	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
6315	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
6316	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
6317	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
6318	Log4(("Host RSP %#RHv\n", uHCReg));
6319	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
6320	Log4(("Host RIP %#RHv\n", uHCReg));
6321	# if HC_ARCH_BITS == 64
6322	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6323	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6324	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6325	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6326	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6327	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6328	# endif
6329	#endif /* VBOX_STRICT */
6330	break;
6331	}
6332
6333	default:
6334	/* Impossible */
6335	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6336	break;
6337	}
6338	}
6339
6340
6341	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
6342	# ifndef VMX_USE_CACHED_VMCS_ACCESSES
6343	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
6344	# endif
6345
6346	/**
6347	* Initialize the VMCS-Read cache.
6348	*
6349	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
6350	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
6351	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
6352	* (those that have a 32-bit FULL & HIGH part).
6353	*
6354	* @param pVCpu The cross context virtual CPU structure.
6355	*/
6356	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
6357	{
6358	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
6359	do { \
6360	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
6361	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
6362	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
6363	++cReadFields; \
6364	} while (0)
6365
6366	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6367	uint32_t cReadFields = 0;
6368
6369	/*
6370	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
6371	* and serve to indicate exceptions to the rules.
6372	*/
6373
6374	/* Guest-natural selector base fields. */
6375	#if 0
6376	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
6377	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
6378	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
6379	#endif
6380	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
6381	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
6382	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
6383	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
6384	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
6385	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
6386	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
6387	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
6388	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
6389	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
6390	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
6391	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
6392	#if 0
6393	/* Unused natural width guest-state fields. */
6394	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
6395	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
6396	#endif
6397	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
6398	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
6399
6400	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
6401	these 64-bit fields (using "FULL" and "HIGH" fields). */
6402	#if 0
6403	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
6404	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
6405	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
6406	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
6407	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
6408	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
6409	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
6410	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
6411	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
6412	#endif
6413
6414	/* Natural width guest-state fields. */
6415	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
6416	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
6417
6418	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6419	{
6420	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
6421	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
6422	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
6423	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
6424	}
6425	else
6426	{
6427	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
6428	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
6429	}
6430
6431	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
6432	}
6433
6434
6435	/**
6436	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
6437	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
6438	* darwin, running 64-bit guests).
6439	*
6440	* @returns VBox status code.
6441	* @param pVCpu The cross context virtual CPU structure.
6442	* @param idxField The VMCS field encoding.
6443	* @param u64Val 16, 32 or 64-bit value.
6444	*/
6445	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6446	{
6447	int rc;
6448	switch (idxField)
6449	{
6450	/*
6451	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
6452	*/
6453	/* 64-bit Control fields. */
6454	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
6455	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
6456	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
6457	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
6458	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
6459	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
6460	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
6461	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
6462	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
6463	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
6464	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
6465	case VMX_VMCS64_CTRL_EPTP_FULL:
6466	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
6467	/* 64-bit Guest-state fields. */
6468	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
6469	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
6470	case VMX_VMCS64_GUEST_PAT_FULL:
6471	case VMX_VMCS64_GUEST_EFER_FULL:
6472	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
6473	case VMX_VMCS64_GUEST_PDPTE0_FULL:
6474	case VMX_VMCS64_GUEST_PDPTE1_FULL:
6475	case VMX_VMCS64_GUEST_PDPTE2_FULL:
6476	case VMX_VMCS64_GUEST_PDPTE3_FULL:
6477	/* 64-bit Host-state fields. */
6478	case VMX_VMCS64_HOST_PAT_FULL:
6479	case VMX_VMCS64_HOST_EFER_FULL:
6480	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
6481	{
6482	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6483	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
6484	break;
6485	}
6486
6487	/*
6488	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
6489	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
6490	*/
6491	/* Natural-width Guest-state fields. */
6492	case VMX_VMCS_GUEST_CR3:
6493	case VMX_VMCS_GUEST_ES_BASE:
6494	case VMX_VMCS_GUEST_CS_BASE:
6495	case VMX_VMCS_GUEST_SS_BASE:
6496	case VMX_VMCS_GUEST_DS_BASE:
6497	case VMX_VMCS_GUEST_FS_BASE:
6498	case VMX_VMCS_GUEST_GS_BASE:
6499	case VMX_VMCS_GUEST_LDTR_BASE:
6500	case VMX_VMCS_GUEST_TR_BASE:
6501	case VMX_VMCS_GUEST_GDTR_BASE:
6502	case VMX_VMCS_GUEST_IDTR_BASE:
6503	case VMX_VMCS_GUEST_RSP:
6504	case VMX_VMCS_GUEST_RIP:
6505	case VMX_VMCS_GUEST_SYSENTER_ESP:
6506	case VMX_VMCS_GUEST_SYSENTER_EIP:
6507	{
6508	if (!(RT_HI_U32(u64Val)))
6509	{
6510	/* If this field is 64-bit, VT-x will zero out the top bits. */
6511	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6512	}
6513	else
6514	{
6515	/* Assert that only the 32->64 switcher case should ever come here. */
6516	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
6517	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
6518	}
6519	break;
6520	}
6521
6522	default:
6523	{
6524	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
6525	pVCpu->hm.s.u32HMError = idxField;
6526	rc = VERR_INVALID_PARAMETER;
6527	break;
6528	}
6529	}
6530	AssertRCReturn(rc, rc);
6531	return rc;
6532	}
6533
6534
6535	/**
6536	* Queue up a VMWRITE by using the VMCS write cache.
6537	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
6538	*
6539	* @param pVCpu The cross context virtual CPU structure.
6540	* @param idxField The VMCS field encoding.
6541	* @param u64Val 16, 32 or 64-bit value.
6542	*/
6543	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6544	{
6545	AssertPtr(pVCpu);
6546	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6547
6548	AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
6549	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
6550
6551	/* Make sure there are no duplicates. */
6552	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
6553	{
6554	if (pCache->Write.aField[i] == idxField)
6555	{
6556	pCache->Write.aFieldVal[i] = u64Val;
6557	return VINF_SUCCESS;
6558	}
6559	}
6560
6561	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
6562	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
6563	pCache->Write.cValidEntries++;
6564	return VINF_SUCCESS;
6565	}
6566	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
6567
6568
6569	/**
6570	* Sets up the usage of TSC-offsetting and updates the VMCS.
6571	*
6572	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6573	* VMX-preemption timer.
6574	*
6575	* @returns VBox status code.
6576	* @param pVCpu The cross context virtual CPU structure.
6577	* @param pVmxTransient The VMX-transient structure.
6578	*
6579	* @remarks No-long-jump zone!!!
6580	*/
6581	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6582	{
6583	bool fOffsettedTsc;
6584	bool fParavirtTsc;
6585	uint64_t uTscOffset;
6586	PVM pVM = pVCpu->CTX_SUFF(pVM);
6587	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6588
6589	if (pVM->hm.s.vmx.fUsePreemptTimer)
6590	{
6591	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6592
6593	/* Make sure the returned values have sane upper and lower boundaries. */
6594	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6595	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6596	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6597	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6598
6599	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6600	* preemption timers here. We probably need to clamp the preemption timer,
6601	* after converting the timer value to the host. */
6602	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6603	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6604	AssertRC(rc);
6605	}
6606	else
6607	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6608
6609	if (fParavirtTsc)
6610	{
6611	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6612	information before every VM-entry, hence disable it for performance sake. */
6613	#if 0
6614	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
6615	AssertRC(rc);
6616	#endif
6617	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
6618	}
6619
6620	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
6621	if ( fOffsettedTsc
6622	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
6623	{
6624	if (pVmxTransient->fIsNestedGuest)
6625	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
6626	if (pVmcsInfo->u64TscOffset != uTscOffset)
6627	{
6628	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
6629	AssertRC(rc);
6630	pVmcsInfo->u64TscOffset = uTscOffset;
6631	}
6632
6633	if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
6634	{
6635	uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
6636	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6637	AssertRC(rc);
6638	pVmcsInfo->u32ProcCtls = uProcCtls;
6639	}
6640	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
6641	}
6642	else
6643	{
6644	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
6645	if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
6646	{
6647	uProcCtls \|= VMX_PROC_CTLS_RDTSC_EXIT;
6648	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6649	AssertRC(rc);
6650	pVmcsInfo->u32ProcCtls = uProcCtls;
6651	}
6652	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
6653	}
6654	}
6655
6656
6657	/**
6658	* Gets the IEM exception flags for the specified vector and IDT vectoring /
6659	* VM-exit interruption info type.
6660	*
6661	* @returns The IEM exception flags.
6662	* @param uVector The event vector.
6663	* @param uVmxEventType The VMX event type.
6664	*
6665	* @remarks This function currently only constructs flags required for
6666	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
6667	* and CR2 aspects of an exception are not included).
6668	*/
6669	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
6670	{
6671	uint32_t fIemXcptFlags;
6672	switch (uVmxEventType)
6673	{
6674	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6675	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6676	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6677	break;
6678
6679	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6680	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6681	break;
6682
6683	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6684	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
6685	break;
6686
6687	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
6688	{
6689	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6690	if (uVector == X86_XCPT_BP)
6691	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
6692	else if (uVector == X86_XCPT_OF)
6693	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
6694	else
6695	{
6696	fIemXcptFlags = 0;
6697	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
6698	}
6699	break;
6700	}
6701
6702	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6703	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6704	break;
6705
6706	default:
6707	fIemXcptFlags = 0;
6708	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
6709	break;
6710	}
6711	return fIemXcptFlags;
6712	}
6713
6714
6715	/**
6716	* Sets an event as a pending event to be injected into the guest.
6717	*
6718	* @param pVCpu The cross context virtual CPU structure.
6719	* @param u32IntInfo The VM-entry interruption-information field.
6720	* @param cbInstr The VM-entry instruction length in bytes (for software
6721	* interrupts, exceptions and privileged software
6722	* exceptions).
6723	* @param u32ErrCode The VM-entry exception error code.
6724	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
6725	* page-fault.
6726	*/
6727	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
6728	RTGCUINTPTR GCPtrFaultAddress)
6729	{
6730	Assert(!pVCpu->hm.s.Event.fPending);
6731	pVCpu->hm.s.Event.fPending = true;
6732	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
6733	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
6734	pVCpu->hm.s.Event.cbInstr = cbInstr;
6735	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
6736	}
6737
6738
6739	/**
6740	* Sets an external interrupt as pending-for-injection into the VM.
6741	*
6742	* @param pVCpu The cross context virtual CPU structure.
6743	* @param u8Interrupt The external interrupt vector.
6744	*/
6745	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
6746	{
6747	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
6748	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6749	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6750	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6751	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6752	}
6753
6754
6755	/**
6756	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
6757	*
6758	* @param pVCpu The cross context virtual CPU structure.
6759	*/
6760	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
6761	{
6762	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
6763	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
6764	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6765	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6766	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6767	}
6768
6769
6770	/**
6771	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
6772	*
6773	* @param pVCpu The cross context virtual CPU structure.
6774	*/
6775	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
6776	{
6777	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
6778	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6779	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6780	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6781	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6782	}
6783
6784
6785	/**
6786	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
6787	*
6788	* @param pVCpu The cross context virtual CPU structure.
6789	*/
6790	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
6791	{
6792	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
6793	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6794	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6795	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6796	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6797	}
6798
6799
6800	/**
6801	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
6802	*
6803	* @param pVCpu The cross context virtual CPU structure.
6804	*/
6805	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
6806	{
6807	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
6808	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6809	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6810	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6811	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6812	}
6813
6814
6815	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6816	/**
6817	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
6818	*
6819	* @param pVCpu The cross context virtual CPU structure.
6820	* @param u32ErrCode The error code for the general-protection exception.
6821	*/
6822	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
6823	{
6824	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
6825	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6826	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6827	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6828	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6829	}
6830
6831
6832	/**
6833	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
6834	*
6835	* @param pVCpu The cross context virtual CPU structure.
6836	* @param u32ErrCode The error code for the stack exception.
6837	*/
6838	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
6839	{
6840	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
6841	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6842	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6843	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6844	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6845	}
6846
6847
6848	/**
6849	* Decodes the memory operand of an instruction that caused a VM-exit.
6850	*
6851	* The VM-exit qualification field provides the displacement field for memory
6852	* operand instructions, if any.
6853	*
6854	* @returns Strict VBox status code (i.e. informational status codes too).
6855	* @retval VINF_SUCCESS if the operand was successfully decoded.
6856	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
6857	* operand.
6858	* @param pVCpu The cross context virtual CPU structure.
6859	* @param uExitInstrInfo The VM-exit instruction information field.
6860	* @param enmMemAccess The memory operand's access type (read or write).
6861	* @param GCPtrDisp The instruction displacement field, if any. For
6862	* RIP-relative addressing pass RIP + displacement here.
6863	* @param pGCPtrMem Where to store the effective destination memory address.
6864	*
6865	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
6866	* virtual-8086 mode hence skips those checks while verifying if the
6867	* segment is valid.
6868	*/
6869	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
6870	PRTGCPTR pGCPtrMem)
6871	{
6872	Assert(pGCPtrMem);
6873	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
6874	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
6875	\| CPUMCTX_EXTRN_CR0);
6876
6877	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
6878	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
6879	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
6880
6881	VMXEXITINSTRINFO ExitInstrInfo;
6882	ExitInstrInfo.u = uExitInstrInfo;
6883	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
6884	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
6885	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
6886	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
6887	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
6888	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
6889	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
6890	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
6891	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
6892
6893	/*
6894	* Validate instruction information.
6895	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
6896	*/
6897	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
6898	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
6899	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
6900	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
6901	AssertLogRelMsgReturn(fIsMemOperand,
6902	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
6903
6904	/*
6905	* Compute the complete effective address.
6906	*
6907	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
6908	* See AMD spec. 4.5.2 "Segment Registers".
6909	*/
6910	RTGCPTR GCPtrMem = GCPtrDisp;
6911	if (fBaseRegValid)
6912	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
6913	if (fIdxRegValid)
6914	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
6915
6916	RTGCPTR const GCPtrOff = GCPtrMem;
6917	if ( !fIsLongMode
6918	\|\| iSegReg >= X86_SREG_FS)
6919	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
6920	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
6921
6922	/*
6923	* Validate effective address.
6924	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
6925	*/
6926	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
6927	Assert(cbAccess > 0);
6928	if (fIsLongMode)
6929	{
6930	if (X86_IS_CANONICAL(GCPtrMem))
6931	{
6932	*pGCPtrMem = GCPtrMem;
6933	return VINF_SUCCESS;
6934	}
6935
6936	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
6937	* "Data Limit Checks in 64-bit Mode". */
6938	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
6939	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6940	return VINF_HM_PENDING_XCPT;
6941	}
6942
6943	/*
6944	* This is a watered down version of iemMemApplySegment().
6945	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
6946	* and segment CPL/DPL checks are skipped.
6947	*/
6948	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
6949	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
6950	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
6951
6952	/* Check if the segment is present and usable. */
6953	if ( pSel->Attr.n.u1Present
6954	&& !pSel->Attr.n.u1Unusable)
6955	{
6956	Assert(pSel->Attr.n.u1DescType);
6957	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
6958	{
6959	/* Check permissions for the data segment. */
6960	if ( enmMemAccess == VMXMEMACCESS_WRITE
6961	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
6962	{
6963	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6964	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
6965	return VINF_HM_PENDING_XCPT;
6966	}
6967
6968	/* Check limits if it's a normal data segment. */
6969	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
6970	{
6971	if ( GCPtrFirst32 > pSel->u32Limit
6972	\|\| GCPtrLast32 > pSel->u32Limit)
6973	{
6974	Log4Func(("Data segment limit exceeded. "
6975	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6976	GCPtrLast32, pSel->u32Limit));
6977	if (iSegReg == X86_SREG_SS)
6978	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6979	else
6980	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6981	return VINF_HM_PENDING_XCPT;
6982	}
6983	}
6984	else
6985	{
6986	/* Check limits if it's an expand-down data segment.
6987	Note! The upper boundary is defined by the B bit, not the G bit! */
6988	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
6989	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
6990	{
6991	Log4Func(("Expand-down data segment limit exceeded. "
6992	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6993	GCPtrLast32, pSel->u32Limit));
6994	if (iSegReg == X86_SREG_SS)
6995	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6996	else
6997	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6998	return VINF_HM_PENDING_XCPT;
6999	}
7000	}
7001	}
7002	else
7003	{
7004	/* Check permissions for the code segment. */
7005	if ( enmMemAccess == VMXMEMACCESS_WRITE
7006	\|\| ( enmMemAccess == VMXMEMACCESS_READ
7007	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
7008	{
7009	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
7010	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
7011	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7012	return VINF_HM_PENDING_XCPT;
7013	}
7014
7015	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
7016	if ( GCPtrFirst32 > pSel->u32Limit
7017	\|\| GCPtrLast32 > pSel->u32Limit)
7018	{
7019	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
7020	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
7021	if (iSegReg == X86_SREG_SS)
7022	hmR0VmxSetPendingXcptSS(pVCpu, 0);
7023	else
7024	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7025	return VINF_HM_PENDING_XCPT;
7026	}
7027	}
7028	}
7029	else
7030	{
7031	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
7032	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7033	return VINF_HM_PENDING_XCPT;
7034	}
7035
7036	*pGCPtrMem = GCPtrMem;
7037	return VINF_SUCCESS;
7038	}
7039
7040
7041	/**
7042	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
7043	* guest attempting to execute a VMX instruction.
7044	*
7045	* @returns Strict VBox status code (i.e. informational status codes too).
7046	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
7047	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
7048	*
7049	* @param pVCpu The cross context virtual CPU structure.
7050	* @param uExitReason The VM-exit reason.
7051	*
7052	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
7053	* @remarks No-long-jump zone!!!
7054	*/
7055	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
7056	{
7057	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
7058	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
7059
7060	/*
7061	* The physical CPU would have already checked the CPU mode/code segment.
7062	* We shall just assert here for paranoia.
7063	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
7064	*/
7065	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
7066	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
7067	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
7068
7069	if (uExitReason == VMX_EXIT_VMXON)
7070	{
7071	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
7072
7073	/*
7074	* We check CR4.VMXE because it is required to be always set while in VMX operation
7075	* by physical CPUs and our CR4 read shadow is only consulted when executing specific
7076	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
7077	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
7078	*/
7079	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
7080	{
7081	Log4Func(("CR4.VMXE is not set -> #UD\n"));
7082	hmR0VmxSetPendingXcptUD(pVCpu);
7083	return VINF_HM_PENDING_XCPT;
7084	}
7085	}
7086	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
7087	{
7088	/*
7089	* The guest has not entered VMX operation but attempted to execute a VMX instruction
7090	* (other than VMXON), we need to raise a #UD.
7091	*/
7092	Log4Func(("Not in VMX root mode -> #UD\n"));
7093	hmR0VmxSetPendingXcptUD(pVCpu);
7094	return VINF_HM_PENDING_XCPT;
7095	}
7096
7097	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
7098	return VINF_SUCCESS;
7099	}
7100	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7101
7102
7103	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7104	{
7105	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7106
7107	/*
7108	* If VT-x marks the segment as unusable, most other bits remain undefined:
7109	* - For CS the L, D and G bits have meaning.
7110	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7111	* - For the remaining data segments no bits are defined.
7112	*
7113	* The present bit and the unusable bit has been observed to be set at the
7114	* same time (the selector was supposed to be invalid as we started executing
7115	* a V8086 interrupt in ring-0).
7116	*
7117	* What should be important for the rest of the VBox code, is that the P bit is
7118	* cleared. Some of the other VBox code recognizes the unusable bit, but
7119	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7120	* safe side here, we'll strip off P and other bits we don't care about. If
7121	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7122	*
7123	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7124	*/
7125	#ifdef VBOX_STRICT
7126	uint32_t const uAttr = pSelReg->Attr.u;
7127	#endif
7128
7129	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7130	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7131	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7132
7133	#ifdef VBOX_STRICT
7134	VMMRZCallRing3Disable(pVCpu);
7135	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7136	# ifdef DEBUG_bird
7137	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7138	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7139	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7140	# endif
7141	VMMRZCallRing3Enable(pVCpu);
7142	NOREF(uAttr);
7143	#endif
7144	RT_NOREF2(pVCpu, idxSel);
7145	}
7146
7147
7148	/**
7149	* Imports a guest segment register from the current VMCS into the guest-CPU
7150	* context.
7151	*
7152	* @returns VBox status code.
7153	* @param pVCpu The cross context virtual CPU structure.
7154	* @param iSegReg The segment register number (X86_SREG_XXX).
7155	*
7156	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7157	* do not log!
7158	*/
7159	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7160	{
7161	Assert(iSegReg < X86_SREG_COUNT);
7162
7163	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7164	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7165	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7166	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
7167	uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
7168	#else
7169	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7170	#endif
7171	uint64_t u64Base;
7172	uint32_t u32Sel, u32Limit, u32Attr;
7173	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7174	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7175	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7176	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
7177	if (RT_SUCCESS(rc))
7178	{
7179	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7180	pSelReg->Sel = u32Sel;
7181	pSelReg->ValidSel = u32Sel;
7182	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7183	pSelReg->u32Limit = u32Limit;
7184	pSelReg->u64Base = u64Base;
7185	pSelReg->Attr.u = u32Attr;
7186	if (u32Attr & X86DESCATTR_UNUSABLE)
7187	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7188	}
7189	return rc;
7190	}
7191
7192
7193	/**
7194	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7195	*
7196	* @returns VBox status code.
7197	* @param pVCpu The cross context virtual CPU structure.
7198	*
7199	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7200	* do not log!
7201	*/
7202	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7203	{
7204	uint64_t u64Base;
7205	uint32_t u32Sel, u32Limit, u32Attr;
7206	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7207	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7208	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7209	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7210	if (RT_SUCCESS(rc))
7211	{
7212	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7213	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7214	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7215	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7216	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7217	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7218	if (u32Attr & X86DESCATTR_UNUSABLE)
7219	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7220	}
7221	return rc;
7222	}
7223
7224
7225	/**
7226	* Imports the guest TR from the current VMCS into the guest-CPU context.
7227	*
7228	* @returns VBox status code.
7229	* @param pVCpu The cross context virtual CPU structure.
7230	*
7231	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7232	* do not log!
7233	*/
7234	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7235	{
7236	uint32_t u32Sel, u32Limit, u32Attr;
7237	uint64_t u64Base;
7238	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7239	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7240	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7241	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7242	AssertRCReturn(rc, rc);
7243
7244	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7245	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7246	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7247	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7248	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7249	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7250	/* TR is the only selector that can never be unusable. */
7251	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7252	return VINF_SUCCESS;
7253	}
7254
7255
7256	/**
7257	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7258	*
7259	* @returns VBox status code.
7260	* @param pVCpu The cross context virtual CPU structure.
7261	*
7262	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7263	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7264	* instead!!!
7265	*/
7266	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7267	{
7268	uint64_t u64Val;
7269	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7270	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7271	{
7272	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7273	if (RT_SUCCESS(rc))
7274	{
7275	pCtx->rip = u64Val;
7276	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7277	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7278	}
7279	return rc;
7280	}
7281	return VINF_SUCCESS;
7282	}
7283
7284
7285	/**
7286	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7287	*
7288	* @returns VBox status code.
7289	* @param pVCpu The cross context virtual CPU structure.
7290	* @param pVmcsInfo The VMCS info. object.
7291	*
7292	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7293	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7294	* instead!!!
7295	*/
7296	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7297	{
7298	uint32_t u32Val;
7299	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7300	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7301	{
7302	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7303	if (RT_SUCCESS(rc))
7304	{
7305	pCtx->eflags.u32 = u32Val;
7306
7307	/* Restore eflags for real-on-v86-mode hack. */
7308	if (pVmcsInfo->RealMode.fRealOnV86Active)
7309	{
7310	pCtx->eflags.Bits.u1VM = 0;
7311	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7312	}
7313	}
7314	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7315	return rc;
7316	}
7317	return VINF_SUCCESS;
7318	}
7319
7320
7321	/**
7322	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7323	* context.
7324	*
7325	* @returns VBox status code.
7326	* @param pVCpu The cross context virtual CPU structure.
7327	* @param pVmcsInfo The VMCS info. object.
7328	*
7329	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7330	* do not log!
7331	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7332	* instead!!!
7333	*/
7334	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7335	{
7336	uint32_t u32Val;
7337	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7338	if (RT_SUCCESS(rc))
7339	{
7340	if (!u32Val)
7341	{
7342	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7343	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7344
7345	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7346	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7347	}
7348	else
7349	{
7350	/*
7351	* We must import RIP here to set our EM interrupt-inhibited state.
7352	* We also import RFLAGS as our code that evaluates pending interrupts
7353	* before VM-entry requires it.
7354	*/
7355	rc = hmR0VmxImportGuestRip(pVCpu);
7356	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7357	if (RT_SUCCESS(rc))
7358	{
7359	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7360	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7361	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7362	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7363
7364	if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
7365	{
7366	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7367	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
7368	}
7369	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7370	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7371	}
7372	}
7373	}
7374	return rc;
7375	}
7376
7377
7378	/**
7379	* Worker for VMXR0ImportStateOnDemand.
7380	*
7381	* @returns VBox status code.
7382	* @param pVCpu The cross context virtual CPU structure.
7383	* @param pVmcsInfo The VMCS info. object.
7384	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7385	*/
7386	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7387	{
7388	#define VMXLOCAL_BREAK_RC(a_rc) \
7389	if (RT_SUCCESS(a_rc)) \
7390	{ } \
7391	else \
7392	break
7393
7394	int rc = VINF_SUCCESS;
7395	PVM pVM = pVCpu->CTX_SUFF(pVM);
7396	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7397	uint64_t u64Val;
7398	uint32_t u32Val;
7399
7400	/*
7401	* Note! This is hack to workaround a mysterious BSOD observed with release builds
7402	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
7403	* neither are other host platforms.
7404	*
7405	* Committing this temporarily as it prevents BSOD.
7406	*
7407	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
7408	*/
7409	#ifdef RT_OS_WINDOWS
7410	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
7411	return VERR_HM_IPE_1;
7412	#endif
7413
7414	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
7415
7416	/*
7417	* We disable interrupts to make the updating of the state and in particular
7418	* the fExtrn modification atomic wrt to preemption hooks.
7419	*/
7420	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7421
7422	fWhat &= pCtx->fExtrn;
7423	if (fWhat)
7424	{
7425	do
7426	{
7427	if (fWhat & CPUMCTX_EXTRN_RIP)
7428	{
7429	rc = hmR0VmxImportGuestRip(pVCpu);
7430	VMXLOCAL_BREAK_RC(rc);
7431	}
7432
7433	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7434	{
7435	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7436	VMXLOCAL_BREAK_RC(rc);
7437	}
7438
7439	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7440	{
7441	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7442	VMXLOCAL_BREAK_RC(rc);
7443	}
7444
7445	if (fWhat & CPUMCTX_EXTRN_RSP)
7446	{
7447	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
7448	VMXLOCAL_BREAK_RC(rc);
7449	pCtx->rsp = u64Val;
7450	}
7451
7452	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7453	{
7454	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7455	if (fWhat & CPUMCTX_EXTRN_CS)
7456	{
7457	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7458	rc \|= hmR0VmxImportGuestRip(pVCpu);
7459	if (fRealOnV86Active)
7460	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7461	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7462	}
7463	if (fWhat & CPUMCTX_EXTRN_SS)
7464	{
7465	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7466	if (fRealOnV86Active)
7467	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7468	}
7469	if (fWhat & CPUMCTX_EXTRN_DS)
7470	{
7471	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7472	if (fRealOnV86Active)
7473	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7474	}
7475	if (fWhat & CPUMCTX_EXTRN_ES)
7476	{
7477	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7478	if (fRealOnV86Active)
7479	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7480	}
7481	if (fWhat & CPUMCTX_EXTRN_FS)
7482	{
7483	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7484	if (fRealOnV86Active)
7485	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7486	}
7487	if (fWhat & CPUMCTX_EXTRN_GS)
7488	{
7489	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7490	if (fRealOnV86Active)
7491	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7492	}
7493	VMXLOCAL_BREAK_RC(rc);
7494	}
7495
7496	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7497	{
7498	if (fWhat & CPUMCTX_EXTRN_LDTR)
7499	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
7500
7501	if (fWhat & CPUMCTX_EXTRN_GDTR)
7502	{
7503	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
7504	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
7505	pCtx->gdtr.pGdt = u64Val;
7506	pCtx->gdtr.cbGdt = u32Val;
7507	}
7508
7509	/* Guest IDTR. */
7510	if (fWhat & CPUMCTX_EXTRN_IDTR)
7511	{
7512	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
7513	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
7514	pCtx->idtr.pIdt = u64Val;
7515	pCtx->idtr.cbIdt = u32Val;
7516	}
7517
7518	/* Guest TR. */
7519	if (fWhat & CPUMCTX_EXTRN_TR)
7520	{
7521	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7522	don't need to import that one. */
7523	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7524	rc \|= hmR0VmxImportGuestTr(pVCpu);
7525	}
7526	VMXLOCAL_BREAK_RC(rc);
7527	}
7528
7529	if (fWhat & CPUMCTX_EXTRN_DR7)
7530	{
7531	if (!pVCpu->hm.s.fUsingHyperDR7)
7532	{
7533	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
7534	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
7535	VMXLOCAL_BREAK_RC(rc);
7536	pCtx->dr[7] = u32Val;
7537	}
7538	}
7539
7540	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7541	{
7542	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
7543	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
7544	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
7545	pCtx->SysEnter.cs = u32Val;
7546	VMXLOCAL_BREAK_RC(rc);
7547	}
7548
7549	#if HC_ARCH_BITS == 64
7550	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7551	{
7552	if ( pVM->hm.s.fAllow64BitGuests
7553	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7554	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7555	}
7556
7557	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7558	{
7559	if ( pVM->hm.s.fAllow64BitGuests
7560	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7561	{
7562	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7563	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7564	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7565	}
7566	}
7567	#endif
7568
7569	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7570	#if HC_ARCH_BITS == 32
7571	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
7572	#endif
7573	)
7574	{
7575	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7576	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7577	Assert(pMsrs);
7578	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
7579	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
7580	for (uint32_t i = 0; i < cMsrs; i++)
7581	{
7582	uint32_t const idMsr = pMsrs[i].u32Msr;
7583	switch (idMsr)
7584	{
7585	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
7586	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
7587	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7588	#if HC_ARCH_BITS == 32
7589	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsrs[i].u64Value; break;
7590	case MSR_K6_STAR: pCtx->msrSTAR = pMsrs[i].u64Value; break;
7591	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsrs[i].u64Value; break;
7592	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsrs[i].u64Value; break;
7593	#endif
7594	default:
7595	{
7596	pCtx->fExtrn = 0;
7597	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
7598	ASMSetFlags(fEFlags);
7599	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
7600	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7601	}
7602	}
7603	}
7604	}
7605
7606	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7607	{
7608	uint64_t u64Shadow;
7609	if (fWhat & CPUMCTX_EXTRN_CR0)
7610	{
7611	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7612	* remove when we drop 32-bit host w/ 64-bit host support, see
7613	* @bugref{9180#c39}. */
7614	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7615	#if HC_ARCH_BITS == 32
7616	uint32_t u32Shadow;
7617	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
7618	u64Shadow = u32Shadow;
7619	#else
7620	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
7621	#endif
7622	VMXLOCAL_BREAK_RC(rc);
7623	u64Val = u32Val;
7624	#if 1
7625	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7626	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7627	#else
7628	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
7629	{
7630	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7631	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7632	}
7633	else
7634	{
7635	/** @todo NSTVMX: We need to do some unfudging here because we altered the
7636	* guest/host mask before running the nested-guest. */
7637	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
7638	Assert(pVmcsNstGst);
7639
7640	uint64_t const uGstCr0Mask = pVmcsNstGst->u64Cr0Mask.u;
7641	uint64_t const uHstCr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
7642	}
7643	#endif
7644	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7645	CPUMSetGuestCR0(pVCpu, u64Val);
7646	VMMRZCallRing3Enable(pVCpu);
7647	}
7648
7649	if (fWhat & CPUMCTX_EXTRN_CR4)
7650	{
7651	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7652	* remove when we drop 32-bit host w/ 64-bit host support, see
7653	* @bugref{9180#c39}. */
7654	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7655	#if HC_ARCH_BITS == 32
7656	uint32_t u32Shadow;
7657	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
7658	u64Shadow = u32Shadow;
7659	#else
7660	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
7661	#endif
7662	VMXLOCAL_BREAK_RC(rc);
7663	u64Val = u32Val;
7664	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
7665	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7666	pCtx->cr4 = u64Val;
7667	}
7668
7669	if (fWhat & CPUMCTX_EXTRN_CR3)
7670	{
7671	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7672	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7673	\|\| ( pVM->hm.s.fNestedPaging
7674	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7675	{
7676	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
7677	VMXLOCAL_BREAK_RC(rc);
7678	if (pCtx->cr3 != u64Val)
7679	{
7680	pCtx->cr3 = u64Val;
7681	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7682	}
7683
7684	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7685	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7686	if (CPUMIsGuestInPAEModeEx(pCtx))
7687	{
7688	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
7689	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
7690	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
7691	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
7692	VMXLOCAL_BREAK_RC(rc);
7693	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7694	}
7695	}
7696	}
7697
7698	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7699	# if 0
7700	/** @todo NSTVMX: We handle most of these fields individually by passing it to IEM
7701	* VM-exit handlers as parameters. We would handle it differently when using
7702	* the fast path. */
7703	/*
7704	* The hardware virtualization state currently consists of VMCS fields that may be
7705	* modified by execution of the nested-guest (that are not part of the general
7706	* guest state) and is visible to guest software. Hence, it is technically part of
7707	* the guest-CPU state when executing a nested-guest.
7708	*/
7709	if ( (fWhat & CPUMCTX_EXTRN_HWVIRT)
7710	&& CPUMIsGuestInVmxNonRootMode(pCtx))
7711	{
7712	PVMXVVMCS pGstVmcs = pCtx->hwvirt.vmx.CTX_SUFF(pVmcs);
7713	rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pGstVmcs->u32RoExitReason);
7714	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pGstVmcs->u64RoExitQual.u);
7715	VMXLOCAL_BREAK_RC(rc);
7716
7717	/*
7718	* VM-entry can fail due to invalid-guest state, machine-check events and
7719	* MSR loading failures. Other than VM-exit reason and VM-exit qualification
7720	* all other VMCS fields are left unmodified on VM-entry failure.
7721	*
7722	* See Intel spec. 26.7 "VM-entry Failures During Or After Loading Guest State".
7723	*/
7724	bool const fEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(pGstVmcs->u32RoExitReason);
7725	if (!fEntryFailed)
7726	{
7727	/*
7728	* Some notes on VMCS fields that may need importing when the fast path
7729	* is implemented. Currently we fully emulate VMLAUNCH/VMRESUME in IEM.
7730	*
7731	* Requires fixing up when using hardware-assisted VMX:
7732	* - VM-exit interruption info: Shouldn't reflect host interrupts/NMIs.
7733	* - VM-exit interruption error code: Cleared to 0 when not appropriate.
7734	* - IDT-vectoring info: Think about this.
7735	* - IDT-vectoring error code: Think about this.
7736	*
7737	* Emulated:
7738	* - Guest-interruptiblity state: Derived from FFs and RIP.
7739	* - Guest pending debug exceptions: Derived from DR6.
7740	* - Guest activity state: Emulated from EM state.
7741	* - Guest PDPTEs: Currently all 0s since we don't support nested EPT.
7742	* - Entry-interrupt info: Emulated, cleared to 0.
7743	*/
7744	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pGstVmcs->u32RoExitIntInfo);
7745	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pGstVmcs->u32RoExitIntErrCode);
7746	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pGstVmcs->u32RoIdtVectoringInfo);
7747	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pGstVmcs->u32RoIdtVectoringErrCode);
7748	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pGstVmcs->u32RoExitInstrLen);
7749	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pGstVmcs->u32RoExitIntInfo);
7750	rc \|= VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pGstVmcs->u64RoGuestPhysAddr.u);
7751	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pGstVmcs->u64RoGuestLinearAddr.u);
7752	/** @todo NSTVMX: Save and adjust preemption timer value. */
7753	}
7754
7755	VMXLOCAL_BREAK_RC(rc);
7756	}
7757	# endif
7758	#endif
7759	}
7760	} while (0);
7761
7762	if (RT_SUCCESS(rc))
7763	{
7764	/* Update fExtrn. */
7765	pCtx->fExtrn &= ~fWhat;
7766
7767	/* If everything has been imported, clear the HM keeper bit. */
7768	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7769	{
7770	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7771	Assert(!pCtx->fExtrn);
7772	}
7773	}
7774	}
7775	else
7776	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7777
7778	ASMSetFlags(fEFlags);
7779
7780	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7781
7782	if (RT_SUCCESS(rc))
7783	{ /* likely */ }
7784	else
7785	return rc;
7786
7787	/*
7788	* Honor any pending CR3 updates.
7789	*
7790	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
7791	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7792	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7793	*
7794	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7795	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7796	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7797	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7798	*
7799	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7800	*/
7801	if (VMMRZCallRing3IsEnabled(pVCpu))
7802	{
7803	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7804	{
7805	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7806	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7807	}
7808
7809	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7810	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7811
7812	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7813	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7814	}
7815
7816	return VINF_SUCCESS;
7817	#undef VMXLOCAL_BREAK_RC
7818	}
7819
7820
7821	/**
7822	* Saves the guest state from the VMCS into the guest-CPU context.
7823	*
7824	* @returns VBox status code.
7825	* @param pVCpu The cross context virtual CPU structure.
7826	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7827	*/
7828	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
7829	{
7830	PCVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7831	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7832	}
7833
7834
7835	/**
7836	* Check per-VM and per-VCPU force flag actions that require us to go back to
7837	* ring-3 for one reason or another.
7838	*
7839	* @returns Strict VBox status code (i.e. informational status codes too)
7840	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7841	* ring-3.
7842	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7843	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7844	* interrupts)
7845	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7846	* all EMTs to be in ring-3.
7847	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7848	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7849	* to the EM loop.
7850	*
7851	* @param pVCpu The cross context virtual CPU structure.
7852	* @param fStepping Whether we are single-stepping the guest using the
7853	* hypervisor debugger.
7854	*/
7855	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
7856	{
7857	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7858
7859	/*
7860	* Update pending interrupts into the APIC's IRR.
7861	*/
7862	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7863	APICUpdatePendingInterrupts(pVCpu);
7864
7865	/*
7866	* Anything pending? Should be more likely than not if we're doing a good job.
7867	*/
7868	PVM pVM = pVCpu->CTX_SUFF(pVM);
7869	if ( !fStepping
7870	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7871	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7872	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7873	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7874	return VINF_SUCCESS;
7875
7876	/* Pending PGM C3 sync. */
7877	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7878	{
7879	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7880	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7881	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7882	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7883	if (rcStrict2 != VINF_SUCCESS)
7884	{
7885	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7886	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7887	return rcStrict2;
7888	}
7889	}
7890
7891	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7892	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7893	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7894	{
7895	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7896	int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7897	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7898	return rc2;
7899	}
7900
7901	/* Pending VM request packets, such as hardware interrupts. */
7902	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7903	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7904	{
7905	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7906	return VINF_EM_PENDING_REQUEST;
7907	}
7908
7909	/* Pending PGM pool flushes. */
7910	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7911	{
7912	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7913	return VINF_PGM_POOL_FLUSH_PENDING;
7914	}
7915
7916	/* Pending DMA requests. */
7917	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7918	{
7919	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7920	return VINF_EM_RAW_TO_R3;
7921	}
7922
7923	return VINF_SUCCESS;
7924	}
7925
7926
7927	/**
7928	* Converts any TRPM trap into a pending HM event. This is typically used when
7929	* entering from ring-3 (not longjmp returns).
7930	*
7931	* @param pVCpu The cross context virtual CPU structure.
7932	*/
7933	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7934	{
7935	Assert(TRPMHasTrap(pVCpu));
7936	Assert(!pVCpu->hm.s.Event.fPending);
7937
7938	uint8_t uVector;
7939	TRPMEVENT enmTrpmEvent;
7940	RTGCUINT uErrCode;
7941	RTGCUINTPTR GCPtrFaultAddress;
7942	uint8_t cbInstr;
7943
7944	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7945	AssertRC(rc);
7946
7947	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7948	uint32_t u32IntInfo = uVector \| VMX_EXIT_INT_INFO_VALID;
7949	if (enmTrpmEvent == TRPM_TRAP)
7950	{
7951	/** @todo r=ramshankar: TRPM currently offers no way to determine a \#DB that was
7952	* generated using INT1 (ICEBP). */
7953	switch (uVector)
7954	{
7955	case X86_XCPT_NMI:
7956	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7957	break;
7958
7959	case X86_XCPT_BP:
7960	case X86_XCPT_OF:
7961	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7962	break;
7963
7964	case X86_XCPT_PF:
7965	case X86_XCPT_DF:
7966	case X86_XCPT_TS:
7967	case X86_XCPT_NP:
7968	case X86_XCPT_SS:
7969	case X86_XCPT_GP:
7970	case X86_XCPT_AC:
7971	u32IntInfo \|= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7972	RT_FALL_THRU();
7973	default:
7974	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7975	break;
7976	}
7977	}
7978	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7979	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7980	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7981	{
7982	switch (uVector)
7983	{
7984	case X86_XCPT_BP:
7985	case X86_XCPT_OF:
7986	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7987	break;
7988
7989	default:
7990	Assert(uVector == X86_XCPT_DB);
7991	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7992	break;
7993	}
7994	}
7995	else
7996	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7997
7998	rc = TRPMResetTrap(pVCpu);
7999	AssertRC(rc);
8000	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
8001	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
8002
8003	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
8004	}
8005
8006
8007	/**
8008	* Converts the pending HM event into a TRPM trap.
8009	*
8010	* @param pVCpu The cross context virtual CPU structure.
8011	*/
8012	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
8013	{
8014	Assert(pVCpu->hm.s.Event.fPending);
8015
8016	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8017	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
8018	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
8019	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
8020
8021	/* If a trap was already pending, we did something wrong! */
8022	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
8023
8024	/** @todo Use HMVmxEventToTrpmEventType() later. */
8025	TRPMEVENT enmTrapType;
8026	switch (uVectorType)
8027	{
8028	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
8029	enmTrapType = TRPM_HARDWARE_INT;
8030	break;
8031
8032	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
8033	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
8034	enmTrapType = TRPM_TRAP;
8035	break;
8036
8037	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT: /* #DB (INT1/ICEBP). */
8038	Assert(uVector == X86_XCPT_DB);
8039	enmTrapType = TRPM_SOFTWARE_INT;
8040	break;
8041
8042	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP (INT3) and #OF (INTO) */
8043	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8044	enmTrapType = TRPM_SOFTWARE_INT;
8045	break;
8046
8047	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
8048	enmTrapType = TRPM_SOFTWARE_INT;
8049	break;
8050
8051	default:
8052	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
8053	enmTrapType = TRPM_32BIT_HACK;
8054	break;
8055	}
8056
8057	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
8058
8059	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
8060	AssertRC(rc);
8061
8062	if (fErrorCodeValid)
8063	TRPMSetErrorCode(pVCpu, uErrorCode);
8064
8065	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8066	&& uVector == X86_XCPT_PF)
8067	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
8068	else if (enmTrapType == TRPM_SOFTWARE_INT)
8069	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
8070
8071	/* We're now done converting the pending event. */
8072	pVCpu->hm.s.Event.fPending = false;
8073	}
8074
8075
8076	/**
8077	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
8078	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
8079	*
8080	* @param pVCpu The cross context virtual CPU structure.
8081	* @param pVmcsInfo The VMCS info. object.
8082	*/
8083	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8084	{
8085	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8086	{
8087	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
8088	{
8089	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
8090	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8091	AssertRC(rc);
8092	}
8093	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
8094	}
8095
8096
8097	/**
8098	* Clears the interrupt-window exiting control in the VMCS.
8099	*
8100	* @param pVmcsInfo The VMCS info. object.
8101	*/
8102	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8103	{
8104	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8105	{
8106	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
8107	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8108	}
8109	return VINF_SUCCESS;
8110	}
8111
8112
8113	/**
8114	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
8115	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
8116	*
8117	* @param pVCpu The cross context virtual CPU structure.
8118	* @param pVmcsInfo The VMCS info. object.
8119	*/
8120	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8121	{
8122	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8123	{
8124	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
8125	{
8126	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8127	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8128	AssertRC(rc);
8129	Log4Func(("Setup NMI-window exiting\n"));
8130	}
8131	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
8132	}
8133
8134
8135	/**
8136	* Clears the NMI-window exiting control in the VMCS.
8137	*
8138	* @param pVmcsInfo The VMCS info. object.
8139	*/
8140	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8141	{
8142	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8143	{
8144	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8145	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8146	}
8147	return VINF_SUCCESS;
8148	}
8149
8150
8151	/**
8152	* Does the necessary state syncing before returning to ring-3 for any reason
8153	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
8154	*
8155	* @returns VBox status code.
8156	* @param pVCpu The cross context virtual CPU structure.
8157	* @param fImportState Whether to import the guest state from the VMCS back
8158	* to the guest-CPU context.
8159	*
8160	* @remarks No-long-jmp zone!!!
8161	*/
8162	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
8163	{
8164	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8165	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8166
8167	RTCPUID idCpu = RTMpCpuId();
8168	Log4Func(("HostCpuId=%u\n", idCpu));
8169
8170	/*
8171	* !!! IMPORTANT !!!
8172	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
8173	*/
8174
8175	/* Save the guest state if necessary. */
8176	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8177	if (fImportState)
8178	{
8179	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8180	AssertRCReturn(rc, rc);
8181	}
8182
8183	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8184	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8185	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8186
8187	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8188	#ifdef VBOX_STRICT
8189	if (CPUMIsHyperDebugStateActive(pVCpu))
8190	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8191	#endif
8192	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8193	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8194	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8195
8196	#if HC_ARCH_BITS == 64
8197	/* Restore host-state bits that VT-x only restores partially. */
8198	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8199	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8200	{
8201	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8202	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8203	}
8204	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8205	#endif
8206
8207	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8208	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8209	{
8210	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8211	if (!fImportState)
8212	{
8213	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8214	AssertRCReturn(rc, rc);
8215	}
8216	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8217	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8218	}
8219	else
8220	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8221
8222	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8223	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8224
8225	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8226	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8227	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8228	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8229	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8230	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8231	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8232	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8233	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8234
8235	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8236
8237	/** @todo This partially defeats the purpose of having preemption hooks.
8238	* The problem is, deregistering the hooks should be moved to a place that
8239	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8240	* context.
8241	*/
8242	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8243	AssertRCReturn(rc, rc);
8244
8245	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8246	NOREF(idCpu);
8247	return VINF_SUCCESS;
8248	}
8249
8250
8251	/**
8252	* Leaves the VT-x session.
8253	*
8254	* @returns VBox status code.
8255	* @param pVCpu The cross context virtual CPU structure.
8256	*
8257	* @remarks No-long-jmp zone!!!
8258	*/
8259	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8260	{
8261	HM_DISABLE_PREEMPT(pVCpu);
8262	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8263	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8264	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8265
8266	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8267	and done this from the VMXR0ThreadCtxCallback(). */
8268	if (!pVCpu->hm.s.fLeaveDone)
8269	{
8270	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8271	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8272	pVCpu->hm.s.fLeaveDone = true;
8273	}
8274	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8275
8276	/*
8277	* !!! IMPORTANT !!!
8278	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8279	*/
8280
8281	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8282	/** @todo Deregistering here means we need to VMCLEAR always
8283	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8284	* for calling VMMR0ThreadCtxHookDisable here! */
8285	VMMR0ThreadCtxHookDisable(pVCpu);
8286
8287	/* Leave HM context. This takes care of local init (term). */
8288	int rc = HMR0LeaveCpu(pVCpu);
8289
8290	HM_RESTORE_PREEMPT();
8291	return rc;
8292	}
8293
8294
8295	/**
8296	* Does the necessary state syncing before doing a longjmp to ring-3.
8297	*
8298	* @returns VBox status code.
8299	* @param pVCpu The cross context virtual CPU structure.
8300	*
8301	* @remarks No-long-jmp zone!!!
8302	*/
8303	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8304	{
8305	return hmR0VmxLeaveSession(pVCpu);
8306	}
8307
8308
8309	/**
8310	* Take necessary actions before going back to ring-3.
8311	*
8312	* An action requires us to go back to ring-3. This function does the necessary
8313	* steps before we can safely return to ring-3. This is not the same as longjmps
8314	* to ring-3, this is voluntary and prepares the guest so it may continue
8315	* executing outside HM (recompiler/IEM).
8316	*
8317	* @returns VBox status code.
8318	* @param pVCpu The cross context virtual CPU structure.
8319	* @param rcExit The reason for exiting to ring-3. Can be
8320	* VINF_VMM_UNKNOWN_RING3_CALL.
8321	*/
8322	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8323	{
8324	Assert(pVCpu);
8325	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8326
8327	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8328	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8329	{
8330	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8331	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8332	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8333	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8334	}
8335
8336	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8337	VMMRZCallRing3Disable(pVCpu);
8338	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8339
8340	/*
8341	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8342	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8343	*
8344	* This is because execution may continue from ring-3 and we would need to inject
8345	* the event from there (hence place it back in TRPM).
8346	*/
8347	if (pVCpu->hm.s.Event.fPending)
8348	{
8349	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8350	Assert(!pVCpu->hm.s.Event.fPending);
8351
8352	/* Clear the events from the VMCS. */
8353	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8354	AssertRCReturn(rc, rc);
8355	}
8356	#ifdef VBOX_STRICT
8357	else
8358	{
8359	/*
8360	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8361	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8362	* occasionally, see @bugref{9180#c42}.
8363	*/
8364	uint32_t uEntryIntInfo;
8365	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8366	AssertRC(rc);
8367	Assert(!VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8368	}
8369	#endif
8370
8371	/*
8372	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8373	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8374	* (e.g. TPR below threshold).
8375	*/
8376	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8377	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8378	AssertRCReturn(rc, rc);
8379
8380	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8381	and if we're injecting an event we should have a TRPM trap pending. */
8382	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8383	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8384	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8385	#endif
8386
8387	/* Save guest state and restore host state bits. */
8388	rc = hmR0VmxLeaveSession(pVCpu);
8389	AssertRCReturn(rc, rc);
8390	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8391
8392	/* Thread-context hooks are unregistered at this point!!! */
8393
8394	/* Sync recompiler state. */
8395	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8396	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8397	\| CPUM_CHANGED_LDTR
8398	\| CPUM_CHANGED_GDTR
8399	\| CPUM_CHANGED_IDTR
8400	\| CPUM_CHANGED_TR
8401	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8402	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8403	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8404	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8405
8406	Assert(!pVCpu->hm.s.fClearTrapFlag);
8407
8408	/* Update the exit-to-ring 3 reason. */
8409	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8410
8411	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8412	if ( rcExit != VINF_EM_RAW_INTERRUPT
8413	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8414	{
8415	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8416	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8417	}
8418
8419	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8420
8421	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
8422	VMMRZCallRing3RemoveNotification(pVCpu);
8423	VMMRZCallRing3Enable(pVCpu);
8424
8425	return rc;
8426	}
8427
8428
8429	/**
8430	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8431	* longjump to ring-3 and possibly get preempted.
8432	*
8433	* @returns VBox status code.
8434	* @param pVCpu The cross context virtual CPU structure.
8435	* @param enmOperation The operation causing the ring-3 longjump.
8436	* @param pvUser User argument, currently unused, NULL.
8437	*/
8438	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
8439	{
8440	RT_NOREF(pvUser);
8441	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8442	{
8443	/*
8444	* !!! IMPORTANT !!!
8445	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8446	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8447	*/
8448	VMMRZCallRing3RemoveNotification(pVCpu);
8449	VMMRZCallRing3Disable(pVCpu);
8450	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
8451	RTThreadPreemptDisable(&PreemptState);
8452
8453	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8454	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8455	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8456	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8457
8458	#if HC_ARCH_BITS == 64
8459	/* Restore host-state bits that VT-x only restores partially. */
8460	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8461	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8462	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8463	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8464	#endif
8465
8466	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8467	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8468	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8469
8470	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8471	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8472	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8473
8474	/* Clear the current VMCS data back to memory. */
8475	hmR0VmxClearVmcs(pVmcsInfo);
8476
8477	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8478	VMMR0ThreadCtxHookDisable(pVCpu);
8479	HMR0LeaveCpu(pVCpu);
8480	RTThreadPreemptRestore(&PreemptState);
8481	return VINF_SUCCESS;
8482	}
8483
8484	Assert(pVCpu);
8485	Assert(pvUser);
8486	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8487	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8488
8489	VMMRZCallRing3Disable(pVCpu);
8490	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8491
8492	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8493
8494	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8495	AssertRCReturn(rc, rc);
8496
8497	VMMRZCallRing3Enable(pVCpu);
8498	return VINF_SUCCESS;
8499	}
8500
8501
8502	/**
8503	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8504	* stack.
8505	*
8506	* @returns Strict VBox status code (i.e. informational status codes too).
8507	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8508	* @param pVCpu The cross context virtual CPU structure.
8509	* @param uValue The value to push to the guest stack.
8510	*/
8511	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
8512	{
8513	/*
8514	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8515	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8516	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8517	*/
8518	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8519	if (pCtx->sp == 1)
8520	return VINF_EM_RESET;
8521	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8522	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8523	AssertRC(rc);
8524	return rc;
8525	}
8526
8527
8528	/**
8529	* Injects an event into the guest upon VM-entry by updating the relevant fields
8530	* in the VM-entry area in the VMCS.
8531	*
8532	* @returns Strict VBox status code (i.e. informational status codes too).
8533	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8534	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8535	*
8536	* @param pVCpu The cross context virtual CPU structure.
8537	* @param pVmxTransient The VMX-transient structure.
8538	* @param pEvent The event being injected.
8539	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state.
8540	* This will be updated if necessary. This cannot not
8541	* be NULL.
8542	* @param fStepping Whether we're single-stepping guest execution and
8543	* should return VINF_EM_DBG_STEPPED if the event is
8544	* injected directly (registers modified by us, not by
8545	* hardware on VM-entry).
8546	*/
8547	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8548	uint32_t *pfIntrState)
8549	{
8550	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8551	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8552	Assert(pfIntrState);
8553
8554	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8555	uint32_t u32IntInfo = pEvent->u64IntInfo;
8556	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8557	uint32_t const cbInstr = pEvent->cbInstr;
8558	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8559	uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8560	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8561
8562	#ifdef VBOX_STRICT
8563	/*
8564	* Validate the error-code-valid bit for hardware exceptions.
8565	* No error codes for exceptions in real-mode.
8566	*
8567	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8568	*/
8569	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8570	&& !CPUMIsGuestInRealModeEx(pCtx))
8571	{
8572	switch (uVector)
8573	{
8574	case X86_XCPT_PF:
8575	case X86_XCPT_DF:
8576	case X86_XCPT_TS:
8577	case X86_XCPT_NP:
8578	case X86_XCPT_SS:
8579	case X86_XCPT_GP:
8580	case X86_XCPT_AC:
8581	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8582	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8583	RT_FALL_THRU();
8584	default:
8585	break;
8586	}
8587	}
8588
8589	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8590	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8591	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8592	#endif
8593
8594	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8595
8596	/*
8597	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8598	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8599	* interrupt handler in the (real-mode) guest.
8600	*
8601	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8602	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8603	*/
8604	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8605	{
8606	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8607	{
8608	/*
8609	* For CPUs with unrestricted guest execution enabled and with the guest
8610	* in real-mode, we must not set the deliver-error-code bit.
8611	*
8612	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8613	*/
8614	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8615	}
8616	else
8617	{
8618	PVM pVM = pVCpu->CTX_SUFF(pVM);
8619	Assert(PDMVmmDevHeapIsEnabled(pVM));
8620	Assert(pVM->hm.s.vmx.pRealModeTSS);
8621	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8622
8623	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8624	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8625	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8626	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8627	AssertRCReturn(rc2, rc2);
8628
8629	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8630	size_t const cbIdtEntry = sizeof(X86IDTR16);
8631	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8632	{
8633	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8634	if (uVector == X86_XCPT_DF)
8635	return VINF_EM_RESET;
8636
8637	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8638	No error codes for exceptions in real-mode. */
8639	if (uVector == X86_XCPT_GP)
8640	{
8641	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8642	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8643	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8644	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8645	HMEVENT EventXcptDf;
8646	RT_ZERO(EventXcptDf);
8647	EventXcptDf.u64IntInfo = uXcptDfInfo;
8648	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8649	}
8650
8651	/*
8652	* If we're injecting an event with no valid IDT entry, inject a #GP.
8653	* No error codes for exceptions in real-mode.
8654	*
8655	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8656	*/
8657	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8658	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8659	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8660	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8661	HMEVENT EventXcptGp;
8662	RT_ZERO(EventXcptGp);
8663	EventXcptGp.u64IntInfo = uXcptGpInfo;
8664	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8665	}
8666
8667	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8668	uint16_t uGuestIp = pCtx->ip;
8669	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8670	{
8671	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8672	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8673	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8674	}
8675	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8676	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8677
8678	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8679	X86IDTR16 IdtEntry;
8680	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8681	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8682	AssertRCReturn(rc2, rc2);
8683
8684	/* Construct the stack frame for the interrupt/exception handler. */
8685	VBOXSTRICTRC rcStrict;
8686	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8687	if (rcStrict == VINF_SUCCESS)
8688	{
8689	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8690	if (rcStrict == VINF_SUCCESS)
8691	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8692	}
8693
8694	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8695	if (rcStrict == VINF_SUCCESS)
8696	{
8697	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8698	pCtx->rip = IdtEntry.offSel;
8699	pCtx->cs.Sel = IdtEntry.uSel;
8700	pCtx->cs.ValidSel = IdtEntry.uSel;
8701	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8702	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8703	&& uVector == X86_XCPT_PF)
8704	pCtx->cr2 = GCPtrFault;
8705
8706	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8707	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8708	\| HM_CHANGED_GUEST_RSP);
8709
8710	/*
8711	* If we delivered a hardware exception (other than an NMI) and if there was
8712	* block-by-STI in effect, we should clear it.
8713	*/
8714	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8715	{
8716	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8717	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8718	Log4Func(("Clearing inhibition due to STI\n"));
8719	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8720	}
8721
8722	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8723	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8724
8725	/*
8726	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8727	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8728	*/
8729	pVCpu->hm.s.Event.fPending = false;
8730
8731	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8732	if (fStepping)
8733	rcStrict = VINF_EM_DBG_STEPPED;
8734	}
8735	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8736	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8737	return rcStrict;
8738	}
8739	}
8740
8741	/*
8742	* Validate.
8743	*/
8744	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8745	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8746
8747	/*
8748	* Inject the event into the VMCS.
8749	*/
8750	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8751	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8752	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8753	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8754	AssertRCReturn(rc, rc);
8755
8756	/*
8757	* Update guest CR2 if this is a page-fault.
8758	*/
8759	if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8760	&& uVector == X86_XCPT_PF)
8761	pCtx->cr2 = GCPtrFault;
8762
8763	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8764	return VINF_SUCCESS;
8765	}
8766
8767
8768	/**
8769	* Evaluates the event to be delivered to the guest and sets it as the pending
8770	* event.
8771	*
8772	* @returns Strict VBox status code (i.e. informational status codes too).
8773	* @param pVCpu The cross context virtual CPU structure.
8774	* @param pVmxTransient The VMX-transient structure.
8775	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8776	*/
8777	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8778	{
8779	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8780	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8781
8782	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
8783	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmcsInfo);
8784	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8785	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8786	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8787
8788	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8789	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8790	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8791	Assert(!TRPMHasTrap(pVCpu));
8792	Assert(pfIntrState);
8793
8794	*pfIntrState = fIntrState;
8795
8796	/*
8797	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
8798	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
8799	*/
8800	/** @todo SMI. SMIs take priority over NMIs. */
8801	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
8802	{
8803	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8804	if ( !pVCpu->hm.s.Event.fPending
8805	&& !fBlockNmi
8806	&& !fBlockSti
8807	&& !fBlockMovSS)
8808	{
8809	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8810	if ( pVmxTransient->fIsNestedGuest
8811	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8812	{
8813	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitNmi(pVCpu);
8814	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8815	return rcStrict;
8816	}
8817	#endif
8818	hmR0VmxSetPendingXcptNmi(pVCpu);
8819	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8820	Log4Func(("Pending NMI\n"));
8821	}
8822	else
8823	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8824	}
8825	/*
8826	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
8827	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
8828	*/
8829	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8830	&& !pVCpu->hm.s.fSingleInstruction)
8831	{
8832	Assert(!DBGFIsStepping(pVCpu));
8833	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8834	AssertRCReturn(rc, rc);
8835	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8836	if ( !pVCpu->hm.s.Event.fPending
8837	&& !fBlockInt
8838	&& !fBlockSti
8839	&& !fBlockMovSS)
8840	{
8841	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8842	if ( pVmxTransient->fIsNestedGuest
8843	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
8844	{
8845	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector /, true / fIntPending */);
8846	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8847	return rcStrict;
8848	}
8849	#endif
8850	uint8_t u8Interrupt;
8851	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8852	if (RT_SUCCESS(rc))
8853	{
8854	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8855	if ( pVmxTransient->fIsNestedGuest
8856	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8857	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8858	{
8859	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8860	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8861	return rcStrict;
8862	}
8863	#endif
8864	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8865	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
8866	}
8867	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8868	{
8869	if ( !pVmxTransient->fIsNestedGuest
8870	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8871	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
8872	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8873
8874	/*
8875	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8876	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8877	* need to re-set this force-flag here.
8878	*/
8879	}
8880	else
8881	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8882	}
8883	else
8884	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8885	}
8886
8887	return VINF_SUCCESS;
8888	}
8889
8890
8891	/**
8892	* Injects any pending events into the guest if the guest is in a state to
8893	* receive them.
8894	*
8895	* @returns Strict VBox status code (i.e. informational status codes too).
8896	* @param pVCpu The cross context virtual CPU structure.
8897	* @param pVmxTransient The VMX-transient structure.
8898	* @param fIntrState The VT-x guest-interruptibility state.
8899	* @param fStepping Whether we are single-stepping the guest using the
8900	* hypervisor debugger and should return
8901	* VINF_EM_DBG_STEPPED if the event was dispatched
8902	* directly.
8903	*/
8904	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
8905	{
8906	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8907	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8908
8909	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8910	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8911
8912	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8913	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8914	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8915	Assert(!TRPMHasTrap(pVCpu));
8916
8917	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8918	if (pVCpu->hm.s.Event.fPending)
8919	{
8920	/*
8921	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8922	* pending even while injecting an event and in this case, we want a VM-exit as soon as
8923	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
8924	*
8925	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8926	*/
8927	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8928	#ifdef VBOX_STRICT
8929	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8930	{
8931	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
8932	Assert(!fBlockInt);
8933	Assert(!fBlockSti);
8934	Assert(!fBlockMovSS);
8935	}
8936	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
8937	{
8938	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8939	Assert(!fBlockSti);
8940	Assert(!fBlockMovSS);
8941	Assert(!fBlockNmi);
8942	}
8943	#endif
8944	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8945	uIntType));
8946
8947	/*
8948	* Inject the event and get any changes to the guest-interruptibility state.
8949	*
8950	* The guest-interruptibility state may need to be updated if we inject the event
8951	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
8952	*/
8953	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
8954	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8955
8956	/*
8957	* If we are executing a nested-guest make sure that we should intercept subsequent
8958	* events. The one we are injecting might be part of VM-entry.
8959	*/
8960	if (pVmxTransient->fIsNestedGuest)
8961	pVCpu->cpum.GstCtx.hwvirt.vmx.fInterceptEvents = true;
8962
8963	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8964	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8965	else
8966	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8967	}
8968
8969	/*
8970	* Update the guest-interruptibility state.
8971	*
8972	* This is required for the real-on-v86 software interrupt injection case above, as well as
8973	* updates to the guest state from ring-3 or IEM/REM.
8974	*/
8975	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
8976	AssertRCReturn(rc, rc);
8977
8978	/*
8979	* There's no need to clear the VM-entry interruption-information field here if we're not
8980	* injecting anything. VT-x clears the valid bit on every VM-exit.
8981	*
8982	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8983	*/
8984
8985	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8986	NOREF(fBlockMovSS); NOREF(fBlockSti);
8987	return rcStrict;
8988	}
8989
8990
8991	/**
8992	* Enters the VT-x session.
8993	*
8994	* @returns VBox status code.
8995	* @param pVCpu The cross context virtual CPU structure.
8996	*/
8997	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
8998	{
8999	AssertPtr(pVCpu);
9000	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
9001	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9002
9003	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9004	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9005	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9006
9007	#ifdef VBOX_STRICT
9008	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
9009	RTCCUINTREG uHostCR4 = ASMGetCR4();
9010	if (!(uHostCR4 & X86_CR4_VMXE))
9011	{
9012	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
9013	return VERR_VMX_X86_CR4_VMXE_CLEARED;
9014	}
9015	#endif
9016
9017	/*
9018	* Load the appropriate VMCS as the current and active one.
9019	*/
9020	PVMXVMCSINFO pVmcsInfo;
9021	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
9022	if (!fInNestedGuestMode)
9023	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
9024	else
9025	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
9026	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
9027	if (RT_SUCCESS(rc))
9028	{
9029	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
9030	pVCpu->hm.s.fLeaveDone = false;
9031	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9032
9033	/*
9034	* Do the EMT scheduled L1D flush here if needed.
9035	*/
9036	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9037	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9038	else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
9039	hmR0MdsClear();
9040	}
9041	return rc;
9042	}
9043
9044
9045	/**
9046	* The thread-context callback (only on platforms which support it).
9047	*
9048	* @param enmEvent The thread-context event.
9049	* @param pVCpu The cross context virtual CPU structure.
9050	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
9051	* @thread EMT(pVCpu)
9052	*/
9053	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
9054	{
9055	NOREF(fGlobalInit);
9056
9057	switch (enmEvent)
9058	{
9059	case RTTHREADCTXEVENT_OUT:
9060	{
9061	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9062	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9063	VMCPU_ASSERT_EMT(pVCpu);
9064
9065	/* No longjmps (logger flushes, locks) in this fragile context. */
9066	VMMRZCallRing3Disable(pVCpu);
9067	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
9068
9069	/* Restore host-state (FPU, debug etc.) */
9070	if (!pVCpu->hm.s.fLeaveDone)
9071	{
9072	/*
9073	* Do -not- import the guest-state here as we might already be in the middle of importing
9074	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
9075	*/
9076	hmR0VmxLeave(pVCpu, false /* fImportState */);
9077	pVCpu->hm.s.fLeaveDone = true;
9078	}
9079
9080	/* Leave HM context, takes care of local init (term). */
9081	int rc = HMR0LeaveCpu(pVCpu);
9082	AssertRC(rc);
9083
9084	/* Restore longjmp state. */
9085	VMMRZCallRing3Enable(pVCpu);
9086	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
9087	break;
9088	}
9089
9090	case RTTHREADCTXEVENT_IN:
9091	{
9092	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9093	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9094	VMCPU_ASSERT_EMT(pVCpu);
9095
9096	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
9097	VMMRZCallRing3Disable(pVCpu);
9098	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
9099
9100	/* Initialize the bare minimum state required for HM. This takes care of
9101	initializing VT-x if necessary (onlined CPUs, local init etc.) */
9102	int rc = hmR0EnterCpu(pVCpu);
9103	AssertRC(rc);
9104	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9105	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9106
9107	/* Load the active VMCS as the current one. */
9108	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9109	rc = hmR0VmxLoadVmcs(pVmcsInfo);
9110	AssertRC(rc);
9111	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9112	pVCpu->hm.s.fLeaveDone = false;
9113
9114	/* Do the EMT scheduled L1D flush if needed. */
9115	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9116	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9117
9118	/* Restore longjmp state. */
9119	VMMRZCallRing3Enable(pVCpu);
9120	break;
9121	}
9122
9123	default:
9124	break;
9125	}
9126	}
9127
9128
9129	/**
9130	* Exports the host state into the VMCS host-state area.
9131	* Sets up the VM-exit MSR-load area.
9132	*
9133	* The CPU state will be loaded from these fields on every successful VM-exit.
9134	*
9135	* @returns VBox status code.
9136	* @param pVCpu The cross context virtual CPU structure.
9137	*
9138	* @remarks No-long-jump zone!!!
9139	*/
9140	static int hmR0VmxExportHostState(PVMCPU pVCpu)
9141	{
9142	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9143
9144	int rc = VINF_SUCCESS;
9145	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
9146	{
9147	rc = hmR0VmxExportHostControlRegs();
9148	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9149
9150	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
9151	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9152
9153	rc = hmR0VmxExportHostMsrs(pVCpu);
9154	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9155
9156	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
9157	}
9158	return rc;
9159	}
9160
9161
9162	/**
9163	* Saves the host state in the VMCS host-state.
9164	*
9165	* @returns VBox status code.
9166	* @param pVCpu The cross context virtual CPU structure.
9167	*
9168	* @remarks No-long-jump zone!!!
9169	*/
9170	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
9171	{
9172	AssertPtr(pVCpu);
9173	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9174
9175	/*
9176	* Export the host state here while entering HM context.
9177	* When thread-context hooks are used, we might get preempted and have to re-save the host
9178	* state but most of the time we won't be, so do it here before we disable interrupts.
9179	*/
9180	return hmR0VmxExportHostState(pVCpu);
9181	}
9182
9183
9184	/**
9185	* Exports the guest state into the VMCS guest-state area.
9186	*
9187	* The will typically be done before VM-entry when the guest-CPU state and the
9188	* VMCS state may potentially be out of sync.
9189	*
9190	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9191	* VM-entry controls.
9192	* Sets up the appropriate VMX non-root function to execute guest code based on
9193	* the guest CPU mode.
9194	*
9195	* @returns VBox strict status code.
9196	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9197	* without unrestricted guest execution and the VMMDev is not presently
9198	* mapped (e.g. EFI32).
9199	*
9200	* @param pVCpu The cross context virtual CPU structure.
9201	* @param pVmxTransient The VMX-transient structure.
9202	*
9203	* @remarks No-long-jump zone!!!
9204	*/
9205	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9206	{
9207	AssertPtr(pVCpu);
9208	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9209	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9210
9211	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9212
9213	/*
9214	* Determine real-on-v86 mode.
9215	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9216	*/
9217	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9218	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9219	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9220	pVmcsInfo->RealMode. fRealOnV86Active = false;
9221	else
9222	{
9223	Assert(!pVmxTransient->fIsNestedGuest);
9224	pVmcsInfo->RealMode.fRealOnV86Active = true;
9225	}
9226
9227	/*
9228	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9229	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9230	*/
9231	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9232	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9233	* be a need to evaluate this everytime since I'm pretty sure we intercept
9234	* all guest paging mode changes. */
9235	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9236	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9237
9238	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9239	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9240
9241	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9242	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9243
9244	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9245	if (rcStrict == VINF_SUCCESS)
9246	{ /* likely */ }
9247	else
9248	{
9249	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9250	return rcStrict;
9251	}
9252
9253	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9254	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9255
9256	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9257	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9258
9259	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9260	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9261
9262	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9263	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9264
9265	rc = hmR0VmxExportGuestRip(pVCpu);
9266	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9267	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9268	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9269
9270	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9271	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9272	\| HM_CHANGED_GUEST_CR2
9273	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9274	\| HM_CHANGED_GUEST_X87
9275	\| HM_CHANGED_GUEST_SSE_AVX
9276	\| HM_CHANGED_GUEST_OTHER_XSAVE
9277	\| HM_CHANGED_GUEST_XCRx
9278	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9279	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9280	\| HM_CHANGED_GUEST_TSC_AUX
9281	\| HM_CHANGED_GUEST_OTHER_MSRS
9282	\| HM_CHANGED_GUEST_HWVIRT /* More accurate PLE handling someday? */
9283	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9284
9285	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9286	return rc;
9287	}
9288
9289
9290	/**
9291	* Exports the state shared between the host and guest into the VMCS.
9292	*
9293	* @param pVCpu The cross context virtual CPU structure.
9294	* @param pVmxTransient The VMX-transient structure.
9295	*
9296	* @remarks No-long-jump zone!!!
9297	*/
9298	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9299	{
9300	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9301	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9302
9303	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9304	{
9305	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9306	AssertRC(rc);
9307	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9308
9309	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9310	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9311	{
9312	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9313	AssertRC(rc);
9314	}
9315	}
9316
9317	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9318	{
9319	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9320	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9321	}
9322
9323	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9324	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9325	}
9326
9327
9328	/**
9329	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9330	*
9331	* @returns Strict VBox status code (i.e. informational status codes too).
9332	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9333	* without unrestricted guest execution and the VMMDev is not presently
9334	* mapped (e.g. EFI32).
9335	*
9336	* @param pVCpu The cross context virtual CPU structure.
9337	* @param pVmxTransient The VMX-transient structure.
9338	*
9339	* @remarks No-long-jump zone!!!
9340	*/
9341	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9342	{
9343	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9344	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9345	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9346
9347	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9348	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9349	#endif
9350
9351	/*
9352	* For many exits it's only RIP that changes and hence try to export it first
9353	* without going through a lot of change flag checks.
9354	*/
9355	VBOXSTRICTRC rcStrict;
9356	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9357	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9358	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9359	{
9360	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9361	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9362	{ /* likely */}
9363	else
9364	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9365	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9366	}
9367	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9368	{
9369	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9370	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9371	{ /* likely */}
9372	else
9373	{
9374	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9375	VBOXSTRICTRC_VAL(rcStrict)));
9376	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9377	return rcStrict;
9378	}
9379	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9380	}
9381	else
9382	rcStrict = VINF_SUCCESS;
9383
9384	#ifdef VBOX_STRICT
9385	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9386	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9387	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9388	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9389	("fCtxChanged=%#RX64\n", fCtxChanged));
9390	#endif
9391	return rcStrict;
9392	}
9393
9394
9395	/**
9396	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9397	* and update error record fields accordingly.
9398	*
9399	* @return VMX_IGS_* return codes.
9400	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9401	* wrong with the guest state.
9402	*
9403	* @param pVCpu The cross context virtual CPU structure.
9404	* @param pVmcsInfo The VMCS info. object.
9405	*
9406	* @remarks This function assumes our cache of the VMCS controls
9407	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9408	*/
9409	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
9410	{
9411	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9412	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9413	uError = (err); \
9414	break; \
9415	} else do { } while (0)
9416
9417	int rc;
9418	PVM pVM = pVCpu->CTX_SUFF(pVM);
9419	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9420	uint32_t uError = VMX_IGS_ERROR;
9421	uint32_t u32Val;
9422	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9423
9424	do
9425	{
9426	/*
9427	* CR0.
9428	*/
9429	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9430	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9431	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
9432	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9433	if (fUnrestrictedGuest)
9434	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
9435
9436	uint32_t u32GuestCr0;
9437	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
9438	AssertRCBreak(rc);
9439	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9440	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9441	if ( !fUnrestrictedGuest
9442	&& (u32GuestCr0 & X86_CR0_PG)
9443	&& !(u32GuestCr0 & X86_CR0_PE))
9444	{
9445	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9446	}
9447
9448	/*
9449	* CR4.
9450	*/
9451	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9452	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9453
9454	uint32_t u32GuestCr4;
9455	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
9456	AssertRCBreak(rc);
9457	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9458	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9459
9460	/*
9461	* IA32_DEBUGCTL MSR.
9462	*/
9463	uint64_t u64Val;
9464	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9465	AssertRCBreak(rc);
9466	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9467	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9468	{
9469	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9470	}
9471	uint64_t u64DebugCtlMsr = u64Val;
9472
9473	#ifdef VBOX_STRICT
9474	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9475	AssertRCBreak(rc);
9476	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9477	#endif
9478	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9479
9480	/*
9481	* RIP and RFLAGS.
9482	*/
9483	uint32_t u32Eflags;
9484	#if HC_ARCH_BITS == 64
9485	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9486	AssertRCBreak(rc);
9487	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9488	if ( !fLongModeGuest
9489	\|\| !pCtx->cs.Attr.n.u1Long)
9490	{
9491	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9492	}
9493	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9494	* must be identical if the "IA-32e mode guest" VM-entry
9495	* control is 1 and CS.L is 1. No check applies if the
9496	* CPU supports 64 linear-address bits. */
9497
9498	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9499	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9500	AssertRCBreak(rc);
9501	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9502	VMX_IGS_RFLAGS_RESERVED);
9503	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9504	u32Eflags = u64Val;
9505	#else
9506	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9507	AssertRCBreak(rc);
9508	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9509	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9510	#endif
9511
9512	if ( fLongModeGuest
9513	\|\| ( fUnrestrictedGuest
9514	&& !(u32GuestCr0 & X86_CR0_PE)))
9515	{
9516	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9517	}
9518
9519	uint32_t u32EntryInfo;
9520	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9521	AssertRCBreak(rc);
9522	if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9523	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9524	{
9525	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9526	}
9527
9528	/*
9529	* 64-bit checks.
9530	*/
9531	#if HC_ARCH_BITS == 64
9532	if (fLongModeGuest)
9533	{
9534	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9535	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9536	}
9537
9538	if ( !fLongModeGuest
9539	&& (u32GuestCr4 & X86_CR4_PCIDE))
9540	{
9541	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9542	}
9543
9544	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9545	* 51:32 beyond the processor's physical-address width are 0. */
9546
9547	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9548	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9549	{
9550	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9551	}
9552
9553	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9554	AssertRCBreak(rc);
9555	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9556
9557	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9558	AssertRCBreak(rc);
9559	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9560	#endif
9561
9562	/*
9563	* PERF_GLOBAL MSR.
9564	*/
9565	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9566	{
9567	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9568	AssertRCBreak(rc);
9569	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9570	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9571	}
9572
9573	/*
9574	* PAT MSR.
9575	*/
9576	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9577	{
9578	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9579	AssertRCBreak(rc);
9580	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9581	for (unsigned i = 0; i < 8; i++)
9582	{
9583	uint8_t u8Val = (u64Val & 0xff);
9584	if ( u8Val != 0 /* UC */
9585	&& u8Val != 1 /* WC */
9586	&& u8Val != 4 /* WT */
9587	&& u8Val != 5 /* WP */
9588	&& u8Val != 6 /* WB */
9589	&& u8Val != 7 /* UC- */)
9590	{
9591	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9592	}
9593	u64Val >>= 8;
9594	}
9595	}
9596
9597	/*
9598	* EFER MSR.
9599	*/
9600	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9601	{
9602	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9603	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9604	AssertRCBreak(rc);
9605	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9606	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9607	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9608	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9609	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9610	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9611	* iemVmxVmentryCheckGuestState(). */
9612	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9613	\|\| !(u32GuestCr0 & X86_CR0_PG)
9614	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9615	VMX_IGS_EFER_LMA_LME_MISMATCH);
9616	}
9617
9618	/*
9619	* Segment registers.
9620	*/
9621	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9622	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9623	if (!(u32Eflags & X86_EFL_VM))
9624	{
9625	/* CS */
9626	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9627	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9628	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9629	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9630	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9631	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9632	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9633	/* CS cannot be loaded with NULL in protected mode. */
9634	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9635	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9636	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9637	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9638	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9639	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9640	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9641	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9642	else
9643	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9644
9645	/* SS */
9646	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9647	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9648	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9649	if ( !(pCtx->cr0 & X86_CR0_PE)
9650	\|\| pCtx->cs.Attr.n.u4Type == 3)
9651	{
9652	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9653	}
9654	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9655	{
9656	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9657	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9658	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9659	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9660	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9661	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9662	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9663	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9664	}
9665
9666	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9667	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9668	{
9669	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9670	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9671	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9672	\|\| pCtx->ds.Attr.n.u4Type > 11
9673	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9674	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9675	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9676	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9677	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9678	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9679	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9680	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9681	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9682	}
9683	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9684	{
9685	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9686	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9687	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9688	\|\| pCtx->es.Attr.n.u4Type > 11
9689	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9690	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9691	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9692	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9693	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9694	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9695	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9696	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9697	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9698	}
9699	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9700	{
9701	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9702	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9703	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9704	\|\| pCtx->fs.Attr.n.u4Type > 11
9705	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9706	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9707	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9708	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9709	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9710	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9711	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9712	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9713	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9714	}
9715	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9716	{
9717	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9718	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9719	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9720	\|\| pCtx->gs.Attr.n.u4Type > 11
9721	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9722	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9723	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9724	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9725	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9726	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9727	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9728	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9729	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9730	}
9731	/* 64-bit capable CPUs. */
9732	#if HC_ARCH_BITS == 64
9733	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9734	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9735	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9736	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9737	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9738	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9739	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9740	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9741	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9742	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9743	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9744	#endif
9745	}
9746	else
9747	{
9748	/* V86 mode checks. */
9749	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9750	if (pVmcsInfo->RealMode.fRealOnV86Active)
9751	{
9752	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9753	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9754	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9755	}
9756	else
9757	{
9758	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9759	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9760	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9761	}
9762
9763	/* CS */
9764	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9765	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9766	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9767	/* SS */
9768	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9769	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9770	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9771	/* DS */
9772	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9773	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9774	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9775	/* ES */
9776	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9777	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9778	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9779	/* FS */
9780	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9781	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9782	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9783	/* GS */
9784	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9785	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9786	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9787	/* 64-bit capable CPUs. */
9788	#if HC_ARCH_BITS == 64
9789	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9790	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9791	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9792	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9793	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9794	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9795	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9796	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9797	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9798	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9799	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9800	#endif
9801	}
9802
9803	/*
9804	* TR.
9805	*/
9806	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9807	/* 64-bit capable CPUs. */
9808	#if HC_ARCH_BITS == 64
9809	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9810	#endif
9811	if (fLongModeGuest)
9812	{
9813	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9814	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9815	}
9816	else
9817	{
9818	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9819	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9820	VMX_IGS_TR_ATTR_TYPE_INVALID);
9821	}
9822	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9823	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9824	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9825	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9826	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9827	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9828	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9829	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9830
9831	/*
9832	* GDTR and IDTR.
9833	*/
9834	#if HC_ARCH_BITS == 64
9835	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9836	AssertRCBreak(rc);
9837	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9838
9839	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9840	AssertRCBreak(rc);
9841	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9842	#endif
9843
9844	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9845	AssertRCBreak(rc);
9846	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9847
9848	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9849	AssertRCBreak(rc);
9850	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9851
9852	/*
9853	* Guest Non-Register State.
9854	*/
9855	/* Activity State. */
9856	uint32_t u32ActivityState;
9857	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9858	AssertRCBreak(rc);
9859	HMVMX_CHECK_BREAK( !u32ActivityState
9860	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9861	VMX_IGS_ACTIVITY_STATE_INVALID);
9862	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9863	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9864	uint32_t u32IntrState;
9865	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9866	AssertRCBreak(rc);
9867	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9868	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9869	{
9870	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9871	}
9872
9873	/** @todo Activity state and injecting interrupts. Left as a todo since we
9874	* currently don't use activity states but ACTIVE. */
9875
9876	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9877	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9878
9879	/* Guest interruptibility-state. */
9880	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9881	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9882	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9883	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9884	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9885	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9886	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9887	if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
9888	{
9889	if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9890	{
9891	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9892	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9893	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9894	}
9895	else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9896	{
9897	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9898	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9899	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9900	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9901	}
9902	}
9903	/** @todo Assumes the processor is not in SMM. */
9904	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9905	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9906	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9907	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9908	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9909	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9910	&& VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9911	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9912	{
9913	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
9914	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9915	}
9916
9917	/* Pending debug exceptions. */
9918	#if HC_ARCH_BITS == 64
9919	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
9920	AssertRCBreak(rc);
9921	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9922	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9923	u32Val = u64Val; /* For pending debug exceptions checks below. */
9924	#else
9925	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
9926	AssertRCBreak(rc);
9927	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9928	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9929	#endif
9930
9931	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9932	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
9933	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9934	{
9935	if ( (u32Eflags & X86_EFL_TF)
9936	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9937	{
9938	/* Bit 14 is PendingDebug.BS. */
9939	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9940	}
9941	if ( !(u32Eflags & X86_EFL_TF)
9942	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9943	{
9944	/* Bit 14 is PendingDebug.BS. */
9945	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9946	}
9947	}
9948
9949	/* VMCS link pointer. */
9950	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9951	AssertRCBreak(rc);
9952	if (u64Val != UINT64_C(0xffffffffffffffff))
9953	{
9954	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9955	/** @todo Bits beyond the processor's physical-address width MBZ. */
9956	/** @todo 32-bit located in memory referenced by value of this field (as a
9957	* physical address) must contain the processor's VMCS revision ID. */
9958	/** @todo SMM checks. */
9959	}
9960
9961	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9962	* not using nested paging? */
9963	if ( pVM->hm.s.fNestedPaging
9964	&& !fLongModeGuest
9965	&& CPUMIsGuestInPAEModeEx(pCtx))
9966	{
9967	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9968	AssertRCBreak(rc);
9969	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9970
9971	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9972	AssertRCBreak(rc);
9973	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9974
9975	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9976	AssertRCBreak(rc);
9977	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9978
9979	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9980	AssertRCBreak(rc);
9981	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9982	}
9983
9984	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9985	if (uError == VMX_IGS_ERROR)
9986	uError = VMX_IGS_REASON_NOT_FOUND;
9987	} while (0);
9988
9989	pVCpu->hm.s.u32HMError = uError;
9990	return uError;
9991
9992	#undef HMVMX_ERROR_BREAK
9993	#undef HMVMX_CHECK_BREAK
9994	}
9995
9996
9997	/**
9998	* Setup the APIC-access page for virtualizing APIC access.
9999	*
10000	* This can cause a longjumps to R3 due to the acquisition of the PGM lock, hence
10001	* this not done as part of exporting guest state, see @bugref{8721}.
10002	*
10003	* @returns VBox status code.
10004	* @param pVCpu The cross context virtual CPU structure.
10005	*/
10006	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
10007	{
10008	PVM pVM = pVCpu->CTX_SUFF(pVM);
10009	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
10010
10011	Assert(PDMHasApic(pVM));
10012	Assert(u64MsrApicBase);
10013
10014	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
10015	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
10016
10017	/* Unalias any existing mapping. */
10018	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
10019	AssertRCReturn(rc, rc);
10020
10021	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
10022	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
10023	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
10024	AssertRCReturn(rc, rc);
10025
10026	/* Update the per-VCPU cache of the APIC base MSR. */
10027	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
10028	return VINF_SUCCESS;
10029	}
10030
10031
10032	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10033	/**
10034	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
10035	* nested-guest using hardware-assisted VMX.
10036	*
10037	* @param pVCpu The cross context virtual CPU structure.
10038	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
10039	* @param pVmcsInfoGst The guest VMCS info. object.
10040	*/
10041	static void hmR0VmxMergeMsrBitmapNested(PCVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
10042	{
10043	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
10044	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
10045	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
10046	Assert(pu64MsrBitmapNstGst);
10047	Assert(pu64MsrBitmapGst);
10048	Assert(pu64MsrBitmap);
10049
10050	/*
10051	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
10052	* MSR that is intercepted by the guest is also intercepted while executing the
10053	* nested-guest using hardware-assisted VMX.
10054	*/
10055	uint32_t const cbFrag = sizeof(uint64_t);
10056	uint32_t const cFrags = X86_PAGE_4K_SIZE / cbFrag;
10057	for (uint32_t i = 0; i <= cFrags; i++)
10058	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
10059	}
10060
10061
10062	/**
10063	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
10064	* hardware-assisted VMX execution of the nested-guest.
10065	*
10066	* For a guest, we don't modify these controls once we set up the VMCS and hence
10067	* this function is never called.
10068	*
10069	* For nested-guests since the guest hypervisor provides these controls on every
10070	* nested-guest VM-entry and could potentially change them everytime we need to
10071	* merge them before every nested-guest VM-entry.
10072	*
10073	* @returns VBox status code.
10074	* @param pVCpu The cross context virtual CPU structure.
10075	*/
10076	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
10077	{
10078	PVM pVM = pVCpu->CTX_SUFF(pVM);
10079	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
10080	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10081	Assert(pVmcsNstGst);
10082
10083	/*
10084	* Merge the controls with the requirements of the guest VMCS.
10085	*
10086	* We do not need to validate the nested-guest VMX features specified in the
10087	* nested-guest VMCS with the features supported by the physical CPU as it's
10088	* already done by the VMLAUNCH/VMRESUME instruction emulation.
10089	*
10090	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the
10091	* guest are derived from the VMX features supported by the physical CPU.
10092	*/
10093
10094	/* Pin-based VM-execution controls. */
10095	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
10096
10097	/* Processor-based VM-execution controls. */
10098	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
10099	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
10100	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
10101	\| VMX_PROC_CTLS_USE_TPR_SHADOW
10102	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
10103
10104	/* Secondary processor-based VM-execution controls. */
10105	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
10106	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
10107	\| VMX_PROC_CTLS2_INVPCID
10108	\| VMX_PROC_CTLS2_RDTSCP
10109	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
10110	\| VMX_PROC_CTLS2_APIC_REG_VIRT
10111	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
10112	\| VMX_PROC_CTLS2_VMFUNC));
10113
10114	/*
10115	* VM-entry controls:
10116	* These controls contains state that depends on the nested-guest state (primarily
10117	* EFER MSR) and is thus not constant through VMLAUNCH/VMRESUME and the nested-guest
10118	* VM-exit. Although the nested-hypervisor cannot change it, we need to in order to
10119	* properly continue executing the nested-guest if the EFER MSR changes but does not
10120	* cause a nested-guest VM-exits.
10121	*
10122	* VM-exit controls:
10123	* These controls specify the host state on return. We cannot use the controls from
10124	* the nested-hypervisor state as is as it would contain the guest state rather than
10125	* the host state. Since the host state is subject to change (e.g. preemption, trips
10126	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
10127	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
10128	*
10129	* VM-entry MSR-load:
10130	* The guest MSRs from the VM-entry MSR-load area are already loaded into the
10131	* guest-CPU context by the VMLAUNCH/VMRESUME instruction emulation.
10132	*
10133	* VM-exit MSR-store:
10134	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU
10135	* context back into the VM-exit MSR-store area.
10136	*
10137	* VM-exit MSR-load areas:
10138	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence,
10139	* we can entirely ignore what the nested-hypervisor wants to load here.
10140	*/
10141
10142	/*
10143	* Exception bitmap.
10144	*
10145	* We could remove #UD from the guest bitmap and merge it with the nested-guest
10146	* bitmap here (and avoid doing anything while exporting nested-guest state), but to
10147	* keep the code more flexible if intercepting exceptions become more dynamic in
10148	* the future we do it as part of exporting the nested-guest state.
10149	*/
10150	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
10151
10152	/*
10153	* CR0/CR4 guest/host mask.
10154	*
10155	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest
10156	* must cause VM-exits, so we need to merge them here.
10157	*/
10158	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
10159	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
10160
10161	/*
10162	* Page-fault error-code mask and match.
10163	*
10164	* Although we require unrestricted guest execution (and thereby nested-paging) for
10165	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
10166	* normally intercept #PFs, it might intercept them for debugging purposes.
10167	*
10168	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF
10169	* filters. If the outer guest is intercepting #PFs we must intercept all #PFs.
10170	*/
10171	uint32_t u32XcptPFMask;
10172	uint32_t u32XcptPFMatch;
10173	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10174	{
10175	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10176	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10177	}
10178	else
10179	{
10180	u32XcptPFMask = 0;
10181	u32XcptPFMatch = 0;
10182	}
10183
10184	/*
10185	* Pause-Loop exiting.
10186	*/
10187	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10188	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10189
10190	/*
10191	* I/O Bitmap.
10192	*
10193	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we
10194	* always intercept all I/O port accesses.
10195	*/
10196	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10197
10198	/*
10199	* APIC-access page.
10200	*
10201	* The APIC-access page address has already been initialized while setting up the
10202	* nested-guest VMCS. In theory, even if the guest-physical address is invalid, it
10203	* should not be on any consequence to the host or to the guest for that matter, but
10204	* we only accept valid addresses verified by the VMLAUNCH/VMRESUME instruction
10205	* emulation to keep it simple.
10206	*/
10207
10208	/*
10209	* Virtual-APIC page and TPR threshold.
10210	*
10211	* We shall use the host-physical address of the virtual-APIC page in guest memory directly.
10212	* For this reason, we can access the virtual-APIC page of the nested-guest only using
10213	* PGM physical handlers as we must not assume a kernel virtual-address mapping exists and
10214	* requesting PGM for a mapping could be expensive/resource intensive (PGM mapping cache).
10215	*/
10216	RTHCPHYS HCPhysVirtApic = NIL_RTHCPHYS;
10217	uint32_t const u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10218	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10219	{
10220	int rc = PGMPhysGCPhys2HCPhys(pVM, pVmcsNstGst->u64AddrVirtApic.u, &HCPhysVirtApic);
10221
10222	/*
10223	* If the guest hypervisor has loaded crap into the virtual-APIC page field
10224	* we would fail to obtain a valid host-physical address for its guest-physical
10225	* address.
10226	*
10227	* We currently do not support this scenario. Maybe in the future if there is a
10228	* pressing need we can explore making this particular set of conditions work.
10229	* Right now we just cause a VM-entry failure.
10230	*
10231	* This has already been checked by VMLAUNCH/VMRESUME instruction emulation,
10232	* so should not really failure at the moment.
10233	*/
10234	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10235	}
10236	else
10237	{
10238	/*
10239	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10240	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10241	* be taken care of by EPT/shadow paging.
10242	*/
10243	if (pVM->hm.s.fAllow64BitGuests)
10244	{
10245	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10246	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10247	}
10248	}
10249
10250	/*
10251	* Validate basic assumptions.
10252	*/
10253	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10254	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10255	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10256	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10257
10258	/*
10259	* Commit it to the nested-guest VMCS.
10260	*/
10261	int rc = VINF_SUCCESS;
10262	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10263	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10264	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10265	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10266	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10267	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10268	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10269	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10270	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10271	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10272	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10273	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10274	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10275	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10276	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10277	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10278	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10279	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10280	{
10281	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10282	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10283	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10284	}
10285	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10286	{
10287	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10288	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10289	}
10290	AssertRCReturn(rc, rc);
10291
10292	/*
10293	* Update the nested-guest VMCS cache.
10294	*/
10295	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10296	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10297	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10298	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10299	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10300	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10301	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10302	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10303	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10304
10305	/*
10306	* MSR bitmap.
10307	*
10308	* The MSR bitmap address has already been initialized while setting up the
10309	* nested-guest VMCS, here we need to merge the MSR bitmaps.
10310	*/
10311	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10312	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10313
10314	return VINF_SUCCESS;
10315	}
10316	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10317
10318
10319	/**
10320	* Does the preparations before executing guest code in VT-x.
10321	*
10322	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10323	* recompiler/IEM. We must be cautious what we do here regarding committing
10324	* guest-state information into the VMCS assuming we assuredly execute the
10325	* guest in VT-x mode.
10326	*
10327	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10328	* the common-state (TRPM/forceflags), we must undo those changes so that the
10329	* recompiler/IEM can (and should) use them when it resumes guest execution.
10330	* Otherwise such operations must be done when we can no longer exit to ring-3.
10331	*
10332	* @returns Strict VBox status code (i.e. informational status codes too).
10333	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10334	* have been disabled.
10335	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10336	* double-fault into the guest.
10337	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10338	* dispatched directly.
10339	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10340	*
10341	* @param pVCpu The cross context virtual CPU structure.
10342	* @param pVmxTransient The VMX-transient structure.
10343	* @param fStepping Whether we are single-stepping the guest in the
10344	* hypervisor debugger. Makes us ignore some of the reasons
10345	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10346	* if event dispatching took place.
10347	*/
10348	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10349	{
10350	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10351
10352	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10353	if (pVmxTransient->fIsNestedGuest)
10354	{
10355	RT_NOREF2(pVCpu, fStepping);
10356	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10357	return VINF_EM_RESCHEDULE_REM;
10358	}
10359	#endif
10360
10361	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10362	PGMRZDynMapFlushAutoSet(pVCpu);
10363	#endif
10364
10365	/*
10366	* Check and process force flag actions, some of which might require us to go back to ring-3.
10367	*/
10368	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10369	if (rcStrict == VINF_SUCCESS)
10370	{ /* FFs don't get set all the time. */ }
10371	else
10372	return rcStrict;
10373
10374	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10375	/*
10376	* Switch to the nested-guest VMCS as we may have transitioned into executing
10377	* the nested-guest without leaving ring-0. Otherwise, if we came from ring-3
10378	* we would load the nested-guest VMCS while entering the VMX ring-0 session.
10379	*
10380	* We do this as late as possible to minimize (though not completely remove)
10381	* clearing/loading VMCS again due to premature trips to ring-3 above.
10382	*/
10383	if (pVmxTransient->fIsNestedGuest)
10384	{
10385	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10386	{
10387	/*
10388	* Ensure we have synced everything from the guest VMCS and also flag that
10389	* that we need to export the full (nested) guest-CPU context to the
10390	* nested-guest VMCS.
10391	*/
10392	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
10393	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST);
10394
10395	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
10396	int rc = hmR0VmxSwitchVmcs(&pVCpu->hm.s.vmx.VmcsInfo, &pVCpu->hm.s.vmx.VmcsInfoNstGst);
10397	if (RT_LIKELY(rc == VINF_SUCCESS))
10398	{
10399	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = true;
10400	ASMSetFlags(fEFlags);
10401	pVmxTransient->pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10402
10403	/*
10404	* We use a different VM-exit MSR-store area for the nested-guest. Hence,
10405	* flag that we need to update the host MSR values there. Even if we decide
10406	* in the future to share the VM-exit MSR-store area page with the guest,
10407	* if its content differs, we would have to update the host MSRs anyway.
10408	*/
10409	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
10410	Assert(!pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer); /** @todo NSTVMX: Paranoia remove later. */
10411	}
10412	else
10413	{
10414	ASMSetFlags(fEFlags);
10415	return rc;
10416	}
10417	}
10418
10419	/*
10420	* Merge guest VMCS controls with the nested-guest VMCS controls.
10421	*
10422	* Even if we have not executed the guest prior to this (e.g. when resuming
10423	* from a saved state), we should be okay with merging controls as we
10424	* initialize the guest VMCS controls as part of VM setup phase.
10425	*/
10426	if (!pVCpu->hm.s.vmx.fMergedNstGstCtls)
10427	{
10428	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10429	AssertRCReturn(rc, rc);
10430	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10431	}
10432	}
10433	#endif
10434
10435	/*
10436	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10437	* We look at the guest VMCS control here as we always set it when supported by
10438	* the physical CPU. Looking at the nested-guest control here would not be
10439	* possible because they are not merged yet.
10440	*/
10441	PVM pVM = pVCpu->CTX_SUFF(pVM);
10442	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10443	Assert(pVmcsInfo);
10444	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10445	&& (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10446	&& PDMHasApic(pVM))
10447	{
10448	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10449	AssertRCReturn(rc, rc);
10450	}
10451
10452	/*
10453	* Evaluate events to be injected into the guest.
10454	*
10455	* Events in TRPM can be injected without inspecting the guest state.
10456	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10457	* guest to cause a VM-exit the next time they are ready to receive the event.
10458	*/
10459	if (TRPMHasTrap(pVCpu))
10460	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10461
10462	uint32_t fIntrState;
10463	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10464
10465	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10466	/*
10467	* While evaluating pending events if something failed (unlikely) or if we were
10468	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10469	*/
10470	if ( rcStrict != VINF_SUCCESS
10471	\|\| ( pVmxTransient->fIsNestedGuest
10472	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)))
10473	return rcStrict;
10474	#endif
10475
10476	/*
10477	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10478	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10479	* also result in triple-faulting the VM.
10480	*
10481	* The above does not apply when executing a nested-guest (since unrestricted guest execution
10482	* is a requirement) regardless doing it avoid duplicating code elsewhere.
10483	*/
10484	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10485	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10486	{ /* likely */ }
10487	else
10488	{
10489	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10490	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10491	return rcStrict;
10492	}
10493
10494	/*
10495	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10496	* import CR3 themselves. We will need to update them here, as even as late as the above
10497	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10498	* the below force flags to be set.
10499	*/
10500	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10501	{
10502	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10503	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10504	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10505	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10506	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10507	}
10508	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10509	{
10510	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10511	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10512	}
10513
10514	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10515	/* Paranoia. */
10516	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10517	#endif
10518
10519	/*
10520	* No longjmps to ring-3 from this point on!!!
10521	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10522	* This also disables flushing of the R0-logger instance (if any).
10523	*/
10524	VMMRZCallRing3Disable(pVCpu);
10525
10526	/*
10527	* Export the guest state bits.
10528	*
10529	* We cannot perform longjmps while loading the guest state because we do not preserve the
10530	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10531	* CPU migration.
10532	*
10533	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10534	* registers. Hence, loading of the guest state needs to be done -after- injection of events.
10535	*/
10536	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10537	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10538	{ /* likely */ }
10539	else
10540	{
10541	VMMRZCallRing3Enable(pVCpu);
10542	return rcStrict;
10543	}
10544
10545	/*
10546	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10547	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10548	* preemption disabled for a while. Since this is purely to aid the
10549	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10550	* disable interrupt on NT.
10551	*
10552	* We need to check for force-flags that could've possible been altered since we last
10553	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10554	* see @bugref{6398}).
10555	*
10556	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10557	* to ring-3 before executing guest code.
10558	*/
10559	pVmxTransient->fEFlags = ASMIntDisableFlags();
10560
10561	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10562	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10563	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10564	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10565	{
10566	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10567	{
10568	pVCpu->hm.s.Event.fPending = false;
10569
10570	/*
10571	* We've injected any pending events. This is really the point of no return (to ring-3).
10572	*
10573	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10574	* returns from this function, so don't enable them here.
10575	*/
10576	return VINF_SUCCESS;
10577	}
10578
10579	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10580	rcStrict = VINF_EM_RAW_INTERRUPT;
10581	}
10582	else
10583	{
10584	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10585	rcStrict = VINF_EM_RAW_TO_R3;
10586	}
10587
10588	ASMSetFlags(pVmxTransient->fEFlags);
10589	VMMRZCallRing3Enable(pVCpu);
10590
10591	return rcStrict;
10592	}
10593
10594
10595	/**
10596	* Final preparations before executing guest code using hardware-assisted VMX.
10597	*
10598	* We can no longer get preempted to a different host CPU and there are no returns
10599	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10600	* failures), this function is not intended to fail sans unrecoverable hardware
10601	* errors.
10602	*
10603	* @param pVCpu The cross context virtual CPU structure.
10604	* @param pVmxTransient The VMX-transient structure.
10605	*
10606	* @remarks Called with preemption disabled.
10607	* @remarks No-long-jump zone!!!
10608	*/
10609	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10610	{
10611	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10612	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10613	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10614	Assert(!pVCpu->hm.s.Event.fPending);
10615
10616	/*
10617	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10618	*/
10619	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10620	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10621
10622	PVM pVM = pVCpu->CTX_SUFF(pVM);
10623	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10624
10625	if (!CPUMIsGuestFPUStateActive(pVCpu))
10626	{
10627	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10628	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10629	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10630	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10631	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10632	}
10633
10634	/*
10635	* Re-save the host state bits as we may've been preempted (only happens when
10636	* thread-context hooks are used or when the VM start function changes).
10637	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10638	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10639	* see @bugref{8432}.
10640	*
10641	* This may also happen when switching to/from a nested-guest VMCS without leaving
10642	* ring-0.
10643	*/
10644	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10645	{
10646	int rc = hmR0VmxExportHostState(pVCpu);
10647	AssertRC(rc);
10648	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
10649	}
10650	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10651
10652	/*
10653	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10654	*/
10655	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10656	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10657	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10658
10659	/*
10660	* Store status of the shared guest/host debug state at the time of VM-entry.
10661	*/
10662	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
10663	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
10664	{
10665	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
10666	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
10667	}
10668	else
10669	#endif
10670	{
10671	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10672	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10673	}
10674
10675	/*
10676	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10677	* more than one conditional check. The post-run side of our code shall determine
10678	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10679	*/
10680	if (pVmcsInfo->pbVirtApic)
10681	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10682
10683	/*
10684	* Update the host MSRs values in the VM-exit MSR-load area.
10685	*/
10686	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10687	{
10688	if (pVmcsInfo->cExitMsrLoad > 0)
10689	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10690	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10691	}
10692
10693	/*
10694	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10695	* VMX-preemption timer based on the next virtual sync clock deadline.
10696	*/
10697	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10698	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10699	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10700	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10701	{
10702	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10703	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10704	}
10705
10706	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10707	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10708	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10709	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
10710
10711	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10712
10713	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10714	as we're about to start executing the guest . */
10715
10716	/*
10717	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10718	*
10719	* This is done this late as updating the TSC offsetting/preemption timer above
10720	* figures out if we can skip intercepting RDTSCP by calculating the number of
10721	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10722	*/
10723	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10724	{
10725	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10726	{
10727	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10728	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
10729	it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
10730	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10731	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10732	AssertRC(rc);
10733	}
10734	else
10735	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
10736	}
10737
10738	#ifdef VBOX_STRICT
10739	Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
10740	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo);
10741	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10742	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo));
10743	#endif
10744
10745	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10746	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10747	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
10748	* see @bugref{9180#c54}. */
10749	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10750	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10751	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10752	#endif
10753	}
10754
10755
10756	/**
10757	* First C routine invoked after running guest code using hardware-assisted VMX.
10758	*
10759	* @param pVCpu The cross context virtual CPU structure.
10760	* @param pVmxTransient The VMX-transient structure.
10761	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10762	*
10763	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10764	*
10765	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10766	* unconditionally when it is safe to do so.
10767	*/
10768	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10769	{
10770	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10771
10772	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10773	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10774	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10775	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10776	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10777	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10778
10779	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10780	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10781	{
10782	uint64_t uGstTsc;
10783	if (!pVmxTransient->fIsNestedGuest)
10784	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10785	else
10786	{
10787	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10788	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10789	}
10790	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10791	}
10792
10793	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10794	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
10795	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10796
10797	#if HC_ARCH_BITS == 64
10798	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10799	#endif
10800	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
10801	/* The 64-on-32 switcher maintains VMCS-launch state on its own
10802	and we need to leave it alone here. */
10803	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
10804	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10805	#else
10806	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10807	#endif
10808	#ifdef VBOX_STRICT
10809	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10810	#endif
10811	Assert(!ASMIntAreEnabled());
10812	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10813	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10814
10815	/*
10816	* Save the basic VM-exit reason and check if the VM-entry failed.
10817	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10818	*/
10819	uint32_t uExitReason;
10820	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
10821	AssertRC(rc);
10822	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
10823	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
10824
10825	/*
10826	* Check if VMLAUNCH/VMRESUME succeeded.
10827	* If this failed, we cause a guru meditation and cease further execution.
10828	*
10829	* However, if we are executing a nested-guest we might fail if we use the
10830	* fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
10831	*/
10832	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
10833	{
10834	/*
10835	* Update the VM-exit history array here even if the VM-entry failed due to:
10836	* - Invalid guest state.
10837	* - MSR loading.
10838	* - Machine-check event.
10839	*
10840	* In any of the above cases we will still have a "valid" VM-exit reason
10841	* despite @a fVMEntryFailed being false.
10842	*
10843	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
10844	*
10845	* Note! We don't have CS or RIP at this point. Will probably address that later
10846	* by amending the history entry added here.
10847	*/
10848	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
10849	UINT64_MAX, uHostTsc);
10850
10851	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
10852	{
10853	VMMRZCallRing3Enable(pVCpu);
10854
10855	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10856	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10857
10858	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
10859	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10860	AssertRC(rc);
10861	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
10862	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
10863	AssertRC(rc);
10864	#else
10865	/*
10866	* Import the guest-interruptibility state always as we need it while evaluating
10867	* injecting events on re-entry.
10868	*
10869	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
10870	* checking for real-mode while exporting the state because all bits that cause
10871	* mode changes wrt CR0 are intercepted.
10872	*/
10873	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
10874	AssertRC(rc);
10875	#endif
10876
10877	/*
10878	* Sync the TPR shadow with our APIC state.
10879	*/
10880	if ( !pVmxTransient->fIsNestedGuest
10881	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
10882	{
10883	Assert(pVmcsInfo->pbVirtApic);
10884	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
10885	{
10886	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
10887	AssertRC(rc);
10888	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
10889	}
10890	}
10891
10892	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10893	return;
10894	}
10895	}
10896	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10897	else if (pVmxTransient->fIsNestedGuest)
10898	{
10899	# if 0
10900	/*
10901	* Copy the VM-instruction error field to the guest VMCS.
10902	*/
10903	/** @todo NSTVMX: Verify we're using the fast path. */
10904	uint32_t u32RoVmInstrError;
10905	rc = VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &u32RoVmInstrError);
10906	AssertRCReturn(rc, rc);
10907	PVMXVVMCS pGstVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10908	pGstVmcs->u32RoVmInstrError = u32RoVmInstrError;
10909	/** @todo NSTVMX: Advance guest RIP and other fast path related restoration. */
10910	# else
10911	AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
10912	# endif
10913	}
10914	#endif
10915	else
10916	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
10917
10918	VMMRZCallRing3Enable(pVCpu);
10919	}
10920
10921
10922	/**
10923	* Runs the guest code using hardware-assisted VMX the normal way.
10924	*
10925	* @returns VBox status code.
10926	* @param pVCpu The cross context virtual CPU structure.
10927	* @param pcLoops Pointer to the number of executed loops.
10928	*/
10929	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
10930	{
10931	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10932	Assert(pcLoops);
10933	Assert(*pcLoops <= cMaxResumeLoops);
10934
10935	VMXTRANSIENT VmxTransient;
10936	RT_ZERO(VmxTransient);
10937	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10938
10939	/* Paranoia. */
10940	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
10941	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10942
10943	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10944	for (;;)
10945	{
10946	Assert(!HMR0SuspendPending());
10947	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10948	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10949
10950	/*
10951	* Preparatory work for running nested-guest code, this may force us to
10952	* return to ring-3.
10953	*
10954	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10955	*/
10956	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10957	if (rcStrict != VINF_SUCCESS)
10958	break;
10959
10960	/* Interrupts are disabled at this point! */
10961	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10962	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10963	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10964	/* Interrupts are re-enabled at this point! */
10965
10966	/*
10967	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10968	*/
10969	if (RT_SUCCESS(rcRun))
10970	{ /* very likely */ }
10971	else
10972	{
10973	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10974	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10975	return rcRun;
10976	}
10977
10978	/*
10979	* Profile the VM-exit.
10980	*/
10981	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10982	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10983	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10984	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10985	HMVMX_START_EXIT_DISPATCH_PROF();
10986
10987	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10988
10989	/*
10990	* Handle the VM-exit.
10991	*/
10992	#ifdef HMVMX_USE_FUNCTION_TABLE
10993	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
10994	#else
10995	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
10996	#endif
10997	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10998	if (rcStrict == VINF_SUCCESS)
10999	{
11000	if (++(*pcLoops) <= cMaxResumeLoops)
11001	continue;
11002	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11003	rcStrict = VINF_EM_RAW_INTERRUPT;
11004	}
11005	break;
11006	}
11007
11008	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11009	return rcStrict;
11010	}
11011
11012	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11013	/**
11014	* Runs the nested-guest code using hardware-assisted VMX.
11015	*
11016	* @returns VBox status code.
11017	* @param pVCpu The cross context virtual CPU structure.
11018	* @param pcLoops Pointer to the number of executed loops.
11019	*
11020	* @sa hmR0VmxRunGuestCodeNormal.
11021	*/
11022	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
11023	{
11024	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11025	Assert(pcLoops);
11026	Assert(*pcLoops <= cMaxResumeLoops);
11027
11028	VMXTRANSIENT VmxTransient;
11029	RT_ZERO(VmxTransient);
11030	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11031	VmxTransient.fIsNestedGuest = true;
11032
11033	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11034	for (;;)
11035	{
11036	Assert(!HMR0SuspendPending());
11037	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11038	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11039
11040	/*
11041	* Preparatory work for running guest code, this may force us to
11042	* return to ring-3.
11043	*
11044	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11045	*/
11046	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11047	if (rcStrict != VINF_SUCCESS)
11048	break;
11049
11050	/* Interrupts are disabled at this point! */
11051	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11052	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11053	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11054	/* Interrupts are re-enabled at this point! */
11055
11056	/*
11057	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11058	*/
11059	if (RT_SUCCESS(rcRun))
11060	{ /* very likely */ }
11061	else
11062	{
11063	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11064	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11065	return rcRun;
11066	}
11067
11068	/*
11069	* Profile the VM-exit.
11070	*/
11071	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11072	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11073	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11074	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11075	HMVMX_START_EXIT_DISPATCH_PROF();
11076
11077	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11078
11079	/*
11080	* Handle the VM-exit.
11081	*/
11082	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
11083	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11084	if ( rcStrict == VINF_SUCCESS
11085	&& CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11086	{
11087	if (++(*pcLoops) <= cMaxResumeLoops)
11088	continue;
11089	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11090	rcStrict = VINF_EM_RAW_INTERRUPT;
11091	}
11092	break;
11093	}
11094
11095	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11096	return rcStrict;
11097	}
11098	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11099
11100
11101	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
11102	* probes.
11103	*
11104	* The following few functions and associated structure contains the bloat
11105	* necessary for providing detailed debug events and dtrace probes as well as
11106	* reliable host side single stepping. This works on the principle of
11107	* "subclassing" the normal execution loop and workers. We replace the loop
11108	* method completely and override selected helpers to add necessary adjustments
11109	* to their core operation.
11110	*
11111	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11112	* any performance for debug and analysis features.
11113	*
11114	* @{
11115	*/
11116
11117	/**
11118	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11119	* the debug run loop.
11120	*/
11121	typedef struct VMXRUNDBGSTATE
11122	{
11123	/** The RIP we started executing at. This is for detecting that we stepped. */
11124	uint64_t uRipStart;
11125	/** The CS we started executing with. */
11126	uint16_t uCsStart;
11127
11128	/** Whether we've actually modified the 1st execution control field. */
11129	bool fModifiedProcCtls : 1;
11130	/** Whether we've actually modified the 2nd execution control field. */
11131	bool fModifiedProcCtls2 : 1;
11132	/** Whether we've actually modified the exception bitmap. */
11133	bool fModifiedXcptBitmap : 1;
11134
11135	/** We desire the modified the CR0 mask to be cleared. */
11136	bool fClearCr0Mask : 1;
11137	/** We desire the modified the CR4 mask to be cleared. */
11138	bool fClearCr4Mask : 1;
11139	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11140	uint32_t fCpe1Extra;
11141	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11142	uint32_t fCpe1Unwanted;
11143	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11144	uint32_t fCpe2Extra;
11145	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11146	uint32_t bmXcptExtra;
11147	/** The sequence number of the Dtrace provider settings the state was
11148	* configured against. */
11149	uint32_t uDtraceSettingsSeqNo;
11150	/** VM-exits to check (one bit per VM-exit). */
11151	uint32_t bmExitsToCheck[3];
11152
11153	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11154	uint32_t fProcCtlsInitial;
11155	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11156	uint32_t fProcCtls2Initial;
11157	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11158	uint32_t bmXcptInitial;
11159	} VMXRUNDBGSTATE;
11160	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11161	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11162
11163
11164	/**
11165	* Initializes the VMXRUNDBGSTATE structure.
11166	*
11167	* @param pVCpu The cross context virtual CPU structure of the
11168	* calling EMT.
11169	* @param pVmxTransient The VMX-transient structure.
11170	* @param pDbgState The debug state to initialize.
11171	*/
11172	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11173	{
11174	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11175	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11176
11177	pDbgState->fModifiedProcCtls = false;
11178	pDbgState->fModifiedProcCtls2 = false;
11179	pDbgState->fModifiedXcptBitmap = false;
11180	pDbgState->fClearCr0Mask = false;
11181	pDbgState->fClearCr4Mask = false;
11182	pDbgState->fCpe1Extra = 0;
11183	pDbgState->fCpe1Unwanted = 0;
11184	pDbgState->fCpe2Extra = 0;
11185	pDbgState->bmXcptExtra = 0;
11186	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11187	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11188	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11189	}
11190
11191
11192	/**
11193	* Updates the VMSC fields with changes requested by @a pDbgState.
11194	*
11195	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11196	* immediately before executing guest code, i.e. when interrupts are disabled.
11197	* We don't check status codes here as we cannot easily assert or return in the
11198	* latter case.
11199	*
11200	* @param pVCpu The cross context virtual CPU structure.
11201	* @param pVmxTransient The VMX-transient structure.
11202	* @param pDbgState The debug state.
11203	*/
11204	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11205	{
11206	/*
11207	* Ensure desired flags in VMCS control fields are set.
11208	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11209	*
11210	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11211	* there should be no stale data in pCtx at this point.
11212	*/
11213	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11214	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11215	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11216	{
11217	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11218	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11219	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11220	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11221	pDbgState->fModifiedProcCtls = true;
11222	}
11223
11224	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11225	{
11226	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11227	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11228	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11229	pDbgState->fModifiedProcCtls2 = true;
11230	}
11231
11232	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11233	{
11234	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11235	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11236	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11237	pDbgState->fModifiedXcptBitmap = true;
11238	}
11239
11240	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11241	{
11242	pVmcsInfo->u64Cr0Mask = 0;
11243	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
11244	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11245	}
11246
11247	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11248	{
11249	pVmcsInfo->u64Cr4Mask = 0;
11250	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
11251	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11252	}
11253
11254	NOREF(pVCpu);
11255	}
11256
11257
11258	/**
11259	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11260	* re-entry next time around.
11261	*
11262	* @returns Strict VBox status code (i.e. informational status codes too).
11263	* @param pVCpu The cross context virtual CPU structure.
11264	* @param pVmxTransient The VMX-transient structure.
11265	* @param pDbgState The debug state.
11266	* @param rcStrict The return code from executing the guest using single
11267	* stepping.
11268	*/
11269	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11270	VBOXSTRICTRC rcStrict)
11271	{
11272	/*
11273	* Restore VM-exit control settings as we may not reenter this function the
11274	* next time around.
11275	*/
11276	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11277
11278	/* We reload the initial value, trigger what we can of recalculations the
11279	next time around. From the looks of things, that's all that's required atm. */
11280	if (pDbgState->fModifiedProcCtls)
11281	{
11282	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11283	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11284	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11285	AssertRCReturn(rc2, rc2);
11286	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11287	}
11288
11289	/* We're currently the only ones messing with this one, so just restore the
11290	cached value and reload the field. */
11291	if ( pDbgState->fModifiedProcCtls2
11292	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11293	{
11294	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11295	AssertRCReturn(rc2, rc2);
11296	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11297	}
11298
11299	/* If we've modified the exception bitmap, we restore it and trigger
11300	reloading and partial recalculation the next time around. */
11301	if (pDbgState->fModifiedXcptBitmap)
11302	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11303
11304	return rcStrict;
11305	}
11306
11307
11308	/**
11309	* Configures VM-exit controls for current DBGF and DTrace settings.
11310	*
11311	* This updates @a pDbgState and the VMCS execution control fields to reflect
11312	* the necessary VM-exits demanded by DBGF and DTrace.
11313	*
11314	* @param pVCpu The cross context virtual CPU structure.
11315	* @param pVmxTransient The VMX-transient structure. May update
11316	* fUpdatedTscOffsettingAndPreemptTimer.
11317	* @param pDbgState The debug state.
11318	*/
11319	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11320	{
11321	/*
11322	* Take down the dtrace serial number so we can spot changes.
11323	*/
11324	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11325	ASMCompilerBarrier();
11326
11327	/*
11328	* We'll rebuild most of the middle block of data members (holding the
11329	* current settings) as we go along here, so start by clearing it all.
11330	*/
11331	pDbgState->bmXcptExtra = 0;
11332	pDbgState->fCpe1Extra = 0;
11333	pDbgState->fCpe1Unwanted = 0;
11334	pDbgState->fCpe2Extra = 0;
11335	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11336	pDbgState->bmExitsToCheck[i] = 0;
11337
11338	/*
11339	* Software interrupts (INT XXh) - no idea how to trigger these...
11340	*/
11341	PVM pVM = pVCpu->CTX_SUFF(pVM);
11342	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11343	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11344	{
11345	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11346	}
11347
11348	/*
11349	* INT3 breakpoints - triggered by #BP exceptions.
11350	*/
11351	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11352	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11353
11354	/*
11355	* Exception bitmap and XCPT events+probes.
11356	*/
11357	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11358	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11359	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11360
11361	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11362	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11363	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11364	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11365	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11366	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11367	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11368	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11369	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11370	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11371	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11372	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11373	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11374	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11375	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11376	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11377	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11378	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11379
11380	if (pDbgState->bmXcptExtra)
11381	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11382
11383	/*
11384	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11385	*
11386	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11387	* So, when adding/changing/removing please don't forget to update it.
11388	*
11389	* Some of the macros are picking up local variables to save horizontal space,
11390	* (being able to see it in a table is the lesser evil here).
11391	*/
11392	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11393	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11394	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11395	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11396	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11397	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11398	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11399	} else do { } while (0)
11400	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11401	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11402	{ \
11403	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11404	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11405	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11406	} else do { } while (0)
11407	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11408	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11409	{ \
11410	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11411	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11412	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11413	} else do { } while (0)
11414	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11415	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11416	{ \
11417	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11418	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11419	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11420	} else do { } while (0)
11421
11422	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11423	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11424	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11425	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11426	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11427
11428	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11429	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11430	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11431	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11432	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11433	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11434	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11435	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11436	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11437	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11438	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11439	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11440	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11441	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11442	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11443	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11444	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11445	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11446	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11447	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11448	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11449	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11450	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11451	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11452	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11453	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11454	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11455	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11456	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11457	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11458	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11459	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11460	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11461	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11462	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11463	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11464
11465	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11466	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11467	{
11468	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11469	\| CPUMCTX_EXTRN_APIC_TPR);
11470	AssertRC(rc);
11471
11472	#if 0 /** @todo fix me */
11473	pDbgState->fClearCr0Mask = true;
11474	pDbgState->fClearCr4Mask = true;
11475	#endif
11476	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11477	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11478	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11479	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11480	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11481	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11482	require clearing here and in the loop if we start using it. */
11483	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11484	}
11485	else
11486	{
11487	if (pDbgState->fClearCr0Mask)
11488	{
11489	pDbgState->fClearCr0Mask = false;
11490	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11491	}
11492	if (pDbgState->fClearCr4Mask)
11493	{
11494	pDbgState->fClearCr4Mask = false;
11495	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11496	}
11497	}
11498	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11499	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11500
11501	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11502	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11503	{
11504	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11505	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11506	}
11507	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11508	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11509
11510	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11511	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11512	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11513	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11514	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11515	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11516	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11517	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11518	#if 0 /** @todo too slow, fix handler. */
11519	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11520	#endif
11521	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11522
11523	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11524	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11525	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11526	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11527	{
11528	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11529	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11530	}
11531	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11532	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11533	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11534	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11535
11536	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11537	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11538	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11539	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11540	{
11541	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11542	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11543	}
11544	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11545	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11546	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11547	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11548
11549	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11550	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11551	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11552	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11553	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11554	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11555	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11556	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11557	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11558	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11559	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11560	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11561	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11562	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11563	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11564	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11565	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11566	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11567	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11568	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11569	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11570	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11571
11572	#undef IS_EITHER_ENABLED
11573	#undef SET_ONLY_XBM_IF_EITHER_EN
11574	#undef SET_CPE1_XBM_IF_EITHER_EN
11575	#undef SET_CPEU_XBM_IF_EITHER_EN
11576	#undef SET_CPE2_XBM_IF_EITHER_EN
11577
11578	/*
11579	* Sanitize the control stuff.
11580	*/
11581	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11582	if (pDbgState->fCpe2Extra)
11583	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11584	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11585	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11586	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11587	{
11588	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11589	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11590	}
11591
11592	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11593	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11594	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11595	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11596	}
11597
11598
11599	/**
11600	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11601	* appropriate.
11602	*
11603	* The caller has checked the VM-exit against the
11604	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11605	* already, so we don't have to do that either.
11606	*
11607	* @returns Strict VBox status code (i.e. informational status codes too).
11608	* @param pVCpu The cross context virtual CPU structure.
11609	* @param pVmxTransient The VMX-transient structure.
11610	* @param uExitReason The VM-exit reason.
11611	*
11612	* @remarks The name of this function is displayed by dtrace, so keep it short
11613	* and to the point. No longer than 33 chars long, please.
11614	*/
11615	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11616	{
11617	/*
11618	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11619	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11620	*
11621	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11622	* does. Must add/change/remove both places. Same ordering, please.
11623	*
11624	* Added/removed events must also be reflected in the next section
11625	* where we dispatch dtrace events.
11626	*/
11627	bool fDtrace1 = false;
11628	bool fDtrace2 = false;
11629	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11630	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11631	uint32_t uEventArg = 0;
11632	#define SET_EXIT(a_EventSubName) \
11633	do { \
11634	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11635	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11636	} while (0)
11637	#define SET_BOTH(a_EventSubName) \
11638	do { \
11639	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11640	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11641	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11642	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11643	} while (0)
11644	switch (uExitReason)
11645	{
11646	case VMX_EXIT_MTF:
11647	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11648
11649	case VMX_EXIT_XCPT_OR_NMI:
11650	{
11651	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11652	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11653	{
11654	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11655	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11656	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11657	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11658	{
11659	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11660	{
11661	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11662	uEventArg = pVmxTransient->uExitIntErrorCode;
11663	}
11664	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11665	switch (enmEvent1)
11666	{
11667	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11668	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11669	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11670	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11671	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11672	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11673	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11674	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11675	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11676	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11677	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11678	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11679	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11680	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11681	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11682	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11683	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11684	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11685	default: break;
11686	}
11687	}
11688	else
11689	AssertFailed();
11690	break;
11691
11692	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11693	uEventArg = idxVector;
11694	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11695	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11696	break;
11697	}
11698	break;
11699	}
11700
11701	case VMX_EXIT_TRIPLE_FAULT:
11702	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11703	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11704	break;
11705	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11706	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11707	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11708	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11709	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11710
11711	/* Instruction specific VM-exits: */
11712	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11713	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11714	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11715	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11716	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11717	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11718	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11719	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11720	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11721	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11722	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11723	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11724	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11725	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11726	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11727	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11728	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11729	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11730	case VMX_EXIT_MOV_CRX:
11731	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11732	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11733	SET_BOTH(CRX_READ);
11734	else
11735	SET_BOTH(CRX_WRITE);
11736	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11737	break;
11738	case VMX_EXIT_MOV_DRX:
11739	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11740	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11741	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11742	SET_BOTH(DRX_READ);
11743	else
11744	SET_BOTH(DRX_WRITE);
11745	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11746	break;
11747	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11748	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11749	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11750	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11751	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11752	case VMX_EXIT_GDTR_IDTR_ACCESS:
11753	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11754	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11755	{
11756	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11757	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11758	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11759	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11760	}
11761	break;
11762
11763	case VMX_EXIT_LDTR_TR_ACCESS:
11764	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11765	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11766	{
11767	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11768	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11769	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11770	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11771	}
11772	break;
11773
11774	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11775	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11776	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11777	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11778	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11779	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11780	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11781	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11782	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11783	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11784	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11785
11786	/* Events that aren't relevant at this point. */
11787	case VMX_EXIT_EXT_INT:
11788	case VMX_EXIT_INT_WINDOW:
11789	case VMX_EXIT_NMI_WINDOW:
11790	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11791	case VMX_EXIT_PREEMPT_TIMER:
11792	case VMX_EXIT_IO_INSTR:
11793	break;
11794
11795	/* Errors and unexpected events. */
11796	case VMX_EXIT_INIT_SIGNAL:
11797	case VMX_EXIT_SIPI:
11798	case VMX_EXIT_IO_SMI:
11799	case VMX_EXIT_SMI:
11800	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11801	case VMX_EXIT_ERR_MSR_LOAD:
11802	case VMX_EXIT_ERR_MACHINE_CHECK:
11803	case VMX_EXIT_PML_FULL:
11804	case VMX_EXIT_VIRTUALIZED_EOI:
11805	break;
11806
11807	default:
11808	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11809	break;
11810	}
11811	#undef SET_BOTH
11812	#undef SET_EXIT
11813
11814	/*
11815	* Dtrace tracepoints go first. We do them here at once so we don't
11816	* have to copy the guest state saving and stuff a few dozen times.
11817	* Down side is that we've got to repeat the switch, though this time
11818	* we use enmEvent since the probes are a subset of what DBGF does.
11819	*/
11820	if (fDtrace1 \|\| fDtrace2)
11821	{
11822	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11823	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11824	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11825	switch (enmEvent1)
11826	{
11827	/** @todo consider which extra parameters would be helpful for each probe. */
11828	case DBGFEVENT_END: break;
11829	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11830	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11831	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11832	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11833	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11834	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11835	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11836	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11837	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
11838	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
11839	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
11840	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
11841	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
11842	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
11843	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
11844	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
11845	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
11846	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
11847	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11848	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11849	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
11850	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
11851	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
11852	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
11853	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
11854	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
11855	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
11856	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11857	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11858	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11859	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11860	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11861	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
11862	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11863	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
11864	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
11865	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
11866	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
11867	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
11868	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
11869	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
11870	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
11871	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
11872	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
11873	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
11874	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
11875	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
11876	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
11877	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
11878	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
11879	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
11880	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
11881	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
11882	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
11883	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
11884	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
11885	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
11886	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
11887	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
11888	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
11889	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
11890	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
11891	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
11892	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
11893	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
11894	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
11895	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
11896	}
11897	switch (enmEvent2)
11898	{
11899	/** @todo consider which extra parameters would be helpful for each probe. */
11900	case DBGFEVENT_END: break;
11901	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
11902	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11903	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
11904	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
11905	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
11906	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
11907	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
11908	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
11909	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
11910	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11911	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11912	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11913	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11914	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11915	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
11916	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11917	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
11918	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
11919	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
11920	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
11921	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
11922	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
11923	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
11924	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
11925	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
11926	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
11927	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
11928	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
11929	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
11930	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
11931	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
11932	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
11933	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
11934	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
11935	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
11936	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
11937	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
11938	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
11939	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
11940	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
11941	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
11942	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
11943	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
11944	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
11945	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
11946	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
11947	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
11948	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
11949	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
11950	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
11951	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
11952	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
11953	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
11954	}
11955	}
11956
11957	/*
11958	* Fire of the DBGF event, if enabled (our check here is just a quick one,
11959	* the DBGF call will do a full check).
11960	*
11961	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
11962	* Note! If we have to events, we prioritize the first, i.e. the instruction
11963	* one, in order to avoid event nesting.
11964	*/
11965	PVM pVM = pVCpu->CTX_SUFF(pVM);
11966	if ( enmEvent1 != DBGFEVENT_END
11967	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
11968	{
11969	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11970	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
11971	if (rcStrict != VINF_SUCCESS)
11972	return rcStrict;
11973	}
11974	else if ( enmEvent2 != DBGFEVENT_END
11975	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
11976	{
11977	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11978	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
11979	if (rcStrict != VINF_SUCCESS)
11980	return rcStrict;
11981	}
11982
11983	return VINF_SUCCESS;
11984	}
11985
11986
11987	/**
11988	* Single-stepping VM-exit filtering.
11989	*
11990	* This is preprocessing the VM-exits and deciding whether we've gotten far
11991	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
11992	* handling is performed.
11993	*
11994	* @returns Strict VBox status code (i.e. informational status codes too).
11995	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
11996	* @param pVmxTransient The VMX-transient structure.
11997	* @param pDbgState The debug state.
11998	*/
11999	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
12000	{
12001	/*
12002	* Expensive (saves context) generic dtrace VM-exit probe.
12003	*/
12004	uint32_t const uExitReason = pVmxTransient->uExitReason;
12005	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
12006	{ /* more likely */ }
12007	else
12008	{
12009	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12010	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12011	AssertRC(rc);
12012	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
12013	}
12014
12015	/*
12016	* Check for host NMI, just to get that out of the way.
12017	*/
12018	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
12019	{ /* normally likely */ }
12020	else
12021	{
12022	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12023	AssertRCReturn(rc2, rc2);
12024	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12025	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12026	return hmR0VmxExitHostNmi(pVCpu);
12027	}
12028
12029	/*
12030	* Check for single stepping event if we're stepping.
12031	*/
12032	if (pVCpu->hm.s.fSingleInstruction)
12033	{
12034	switch (uExitReason)
12035	{
12036	case VMX_EXIT_MTF:
12037	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12038
12039	/* Various events: */
12040	case VMX_EXIT_XCPT_OR_NMI:
12041	case VMX_EXIT_EXT_INT:
12042	case VMX_EXIT_TRIPLE_FAULT:
12043	case VMX_EXIT_INT_WINDOW:
12044	case VMX_EXIT_NMI_WINDOW:
12045	case VMX_EXIT_TASK_SWITCH:
12046	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12047	case VMX_EXIT_APIC_ACCESS:
12048	case VMX_EXIT_EPT_VIOLATION:
12049	case VMX_EXIT_EPT_MISCONFIG:
12050	case VMX_EXIT_PREEMPT_TIMER:
12051
12052	/* Instruction specific VM-exits: */
12053	case VMX_EXIT_CPUID:
12054	case VMX_EXIT_GETSEC:
12055	case VMX_EXIT_HLT:
12056	case VMX_EXIT_INVD:
12057	case VMX_EXIT_INVLPG:
12058	case VMX_EXIT_RDPMC:
12059	case VMX_EXIT_RDTSC:
12060	case VMX_EXIT_RSM:
12061	case VMX_EXIT_VMCALL:
12062	case VMX_EXIT_VMCLEAR:
12063	case VMX_EXIT_VMLAUNCH:
12064	case VMX_EXIT_VMPTRLD:
12065	case VMX_EXIT_VMPTRST:
12066	case VMX_EXIT_VMREAD:
12067	case VMX_EXIT_VMRESUME:
12068	case VMX_EXIT_VMWRITE:
12069	case VMX_EXIT_VMXOFF:
12070	case VMX_EXIT_VMXON:
12071	case VMX_EXIT_MOV_CRX:
12072	case VMX_EXIT_MOV_DRX:
12073	case VMX_EXIT_IO_INSTR:
12074	case VMX_EXIT_RDMSR:
12075	case VMX_EXIT_WRMSR:
12076	case VMX_EXIT_MWAIT:
12077	case VMX_EXIT_MONITOR:
12078	case VMX_EXIT_PAUSE:
12079	case VMX_EXIT_GDTR_IDTR_ACCESS:
12080	case VMX_EXIT_LDTR_TR_ACCESS:
12081	case VMX_EXIT_INVEPT:
12082	case VMX_EXIT_RDTSCP:
12083	case VMX_EXIT_INVVPID:
12084	case VMX_EXIT_WBINVD:
12085	case VMX_EXIT_XSETBV:
12086	case VMX_EXIT_RDRAND:
12087	case VMX_EXIT_INVPCID:
12088	case VMX_EXIT_VMFUNC:
12089	case VMX_EXIT_RDSEED:
12090	case VMX_EXIT_XSAVES:
12091	case VMX_EXIT_XRSTORS:
12092	{
12093	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12094	AssertRCReturn(rc, rc);
12095	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
12096	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
12097	return VINF_EM_DBG_STEPPED;
12098	break;
12099	}
12100
12101	/* Errors and unexpected events: */
12102	case VMX_EXIT_INIT_SIGNAL:
12103	case VMX_EXIT_SIPI:
12104	case VMX_EXIT_IO_SMI:
12105	case VMX_EXIT_SMI:
12106	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12107	case VMX_EXIT_ERR_MSR_LOAD:
12108	case VMX_EXIT_ERR_MACHINE_CHECK:
12109	case VMX_EXIT_PML_FULL:
12110	case VMX_EXIT_VIRTUALIZED_EOI:
12111	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12112	break;
12113
12114	default:
12115	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12116	break;
12117	}
12118	}
12119
12120	/*
12121	* Check for debugger event breakpoints and dtrace probes.
12122	*/
12123	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12124	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12125	{
12126	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12127	if (rcStrict != VINF_SUCCESS)
12128	return rcStrict;
12129	}
12130
12131	/*
12132	* Normal processing.
12133	*/
12134	#ifdef HMVMX_USE_FUNCTION_TABLE
12135	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
12136	#else
12137	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
12138	#endif
12139	}
12140
12141
12142	/**
12143	* Single steps guest code using hardware-assisted VMX.
12144	*
12145	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
12146	* but single-stepping through the hypervisor debugger.
12147	*
12148	* @returns Strict VBox status code (i.e. informational status codes too).
12149	* @param pVCpu The cross context virtual CPU structure.
12150	* @param pcLoops Pointer to the number of executed loops.
12151	*
12152	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
12153	*/
12154	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
12155	{
12156	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
12157	Assert(pcLoops);
12158	Assert(*pcLoops <= cMaxResumeLoops);
12159
12160	VMXTRANSIENT VmxTransient;
12161	RT_ZERO(VmxTransient);
12162	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
12163
12164	/* Set HMCPU indicators. */
12165	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
12166	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
12167	pVCpu->hm.s.fDebugWantRdTscExit = false;
12168	pVCpu->hm.s.fUsingDebugLoop = true;
12169
12170	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
12171	VMXRUNDBGSTATE DbgState;
12172	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
12173	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12174
12175	/*
12176	* The loop.
12177	*/
12178	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
12179	for (;;)
12180	{
12181	Assert(!HMR0SuspendPending());
12182	HMVMX_ASSERT_CPU_SAFE(pVCpu);
12183	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12184	bool fStepping = pVCpu->hm.s.fSingleInstruction;
12185
12186	/* Set up VM-execution controls the next two can respond to. */
12187	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12188
12189	/*
12190	* Preparatory work for running guest code, this may force us to
12191	* return to ring-3.
12192	*
12193	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12194	*/
12195	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12196	if (rcStrict != VINF_SUCCESS)
12197	break;
12198
12199	/* Interrupts are disabled at this point! */
12200	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12201
12202	/* Override any obnoxious code in the above two calls. */
12203	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12204
12205	/*
12206	* Finally execute the guest.
12207	*/
12208	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12209
12210	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12211	/* Interrupts are re-enabled at this point! */
12212
12213	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12214	if (RT_SUCCESS(rcRun))
12215	{ /* very likely */ }
12216	else
12217	{
12218	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12219	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12220	return rcRun;
12221	}
12222
12223	/* Profile the VM-exit. */
12224	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12225	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12226	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12227	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12228	HMVMX_START_EXIT_DISPATCH_PROF();
12229
12230	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12231
12232	/*
12233	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12234	*/
12235	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12236	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12237	if (rcStrict != VINF_SUCCESS)
12238	break;
12239	if (++(*pcLoops) > cMaxResumeLoops)
12240	{
12241	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12242	rcStrict = VINF_EM_RAW_INTERRUPT;
12243	break;
12244	}
12245
12246	/*
12247	* Stepping: Did the RIP change, if so, consider it a single step.
12248	* Otherwise, make sure one of the TFs gets set.
12249	*/
12250	if (fStepping)
12251	{
12252	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12253	AssertRC(rc);
12254	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12255	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12256	{
12257	rcStrict = VINF_EM_DBG_STEPPED;
12258	break;
12259	}
12260	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12261	}
12262
12263	/*
12264	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12265	*/
12266	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12267	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12268	}
12269
12270	/*
12271	* Clear the X86_EFL_TF if necessary.
12272	*/
12273	if (pVCpu->hm.s.fClearTrapFlag)
12274	{
12275	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12276	AssertRC(rc);
12277	pVCpu->hm.s.fClearTrapFlag = false;
12278	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12279	}
12280	/** @todo there seems to be issues with the resume flag when the monitor trap
12281	* flag is pending without being used. Seen early in bios init when
12282	* accessing APIC page in protected mode. */
12283
12284	/*
12285	* Restore VM-exit control settings as we may not re-enter this function the
12286	* next time around.
12287	*/
12288	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12289
12290	/* Restore HMCPU indicators. */
12291	pVCpu->hm.s.fUsingDebugLoop = false;
12292	pVCpu->hm.s.fDebugWantRdTscExit = false;
12293	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12294
12295	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12296	return rcStrict;
12297	}
12298
12299
12300	/** @} */
12301
12302
12303	/**
12304	* Checks if any expensive dtrace probes are enabled and we should go to the
12305	* debug loop.
12306	*
12307	* @returns true if we should use debug loop, false if not.
12308	*/
12309	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12310	{
12311	/* It's probably faster to OR the raw 32-bit counter variables together.
12312	Since the variables are in an array and the probes are next to one
12313	another (more or less), we have good locality. So, better read
12314	eight-nine cache lines ever time and only have one conditional, than
12315	128+ conditionals, right? */
12316	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12317	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12318	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12319	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12320	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12321	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12322	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12323	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12324	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12325	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12326	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12327	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12328	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12329	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12330	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12331	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12332	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12333	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12334	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12335	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12336	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12337	) != 0
12338	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12339	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12340	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12341	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12342	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12343	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12344	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12345	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12346	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12347	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12348	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12349	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12350	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12351	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12352	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12353	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12354	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12355	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12356	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12357	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12358	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12359	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12360	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12361	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12362	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12363	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12364	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12365	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12366	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12367	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12368	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12369	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12370	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12371	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12372	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12373	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12374	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12375	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12376	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12377	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12378	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12379	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12380	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12381	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12382	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12383	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12384	) != 0
12385	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12386	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12387	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12388	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12389	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12390	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12391	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12392	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12393	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12394	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12395	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12396	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12397	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12398	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12399	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12400	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12401	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12402	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12403	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12404	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12405	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12406	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12407	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12408	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12409	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12410	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12411	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12412	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12413	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12414	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12415	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12416	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12417	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12418	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12419	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12420	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12421	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12422	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12423	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12424	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12425	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12426	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12427	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12428	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12429	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12430	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12431	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12432	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12433	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12434	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12435	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12436	) != 0;
12437	}
12438
12439
12440	/**
12441	* Runs the guest using hardware-assisted VMX.
12442	*
12443	* @returns Strict VBox status code (i.e. informational status codes too).
12444	* @param pVCpu The cross context virtual CPU structure.
12445	*/
12446	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
12447	{
12448	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12449	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12450	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12451	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12452
12453	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
12454
12455	VBOXSTRICTRC rcStrict;
12456	uint32_t cLoops = 0;
12457	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12458	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12459	#else
12460	bool const fInNestedGuestMode = false;
12461	#endif
12462	if (!fInNestedGuestMode)
12463	{
12464	if ( !pVCpu->hm.s.fUseDebugLoop
12465	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12466	&& !DBGFIsStepping(pVCpu)
12467	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12468	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12469	else
12470	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12471	}
12472	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12473	else
12474	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
12475
12476	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12477	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12478	#endif
12479
12480	if (rcStrict == VERR_EM_INTERPRETER)
12481	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12482	else if (rcStrict == VINF_EM_RESET)
12483	rcStrict = VINF_EM_TRIPLE_FAULT;
12484
12485	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12486	if (RT_FAILURE(rc2))
12487	{
12488	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12489	rcStrict = rc2;
12490	}
12491	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12492	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12493	return rcStrict;
12494	}
12495
12496
12497	#ifndef HMVMX_USE_FUNCTION_TABLE
12498	/**
12499	* Handles a guest VM-exit from hardware-assisted VMX execution.
12500	*
12501	* @returns Strict VBox status code (i.e. informational status codes too).
12502	* @param pVCpu The cross context virtual CPU structure.
12503	* @param pVmxTransient The VMX-transient structure.
12504	*/
12505	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12506	{
12507	#ifdef DEBUG_ramshankar
12508	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12509	do { \
12510	if (a_fSave != 0) \
12511	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
12512	VBOXSTRICTRC rcStrict = a_CallExpr; \
12513	if (a_fSave != 0) \
12514	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12515	return rcStrict; \
12516	} while (0)
12517	#else
12518	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12519	#endif
12520	uint32_t const uExitReason = pVmxTransient->uExitReason;
12521	switch (uExitReason)
12522	{
12523	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12524	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12525	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12526	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12527	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12528	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12529	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12530	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12531	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12532	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12533	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12534	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12535	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12536	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12537	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12538	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12539	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12540	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12541	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12542	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12543	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12544	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12545	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12546	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12547	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12548	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12549	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12550	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12551	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12552	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12553	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12554	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12555	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12556	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12557	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12558	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12559	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12560	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12561	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12562	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12563	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
12564	case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient));
12565	#else
12566	case VMX_EXIT_VMCLEAR:
12567	case VMX_EXIT_VMLAUNCH:
12568	case VMX_EXIT_VMPTRLD:
12569	case VMX_EXIT_VMPTRST:
12570	case VMX_EXIT_VMREAD:
12571	case VMX_EXIT_VMRESUME:
12572	case VMX_EXIT_VMWRITE:
12573	case VMX_EXIT_VMXOFF:
12574	case VMX_EXIT_VMXON:
12575	case VMX_EXIT_INVVPID:
12576	case VMX_EXIT_INVEPT:
12577	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12578	#endif
12579
12580	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12581	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12582	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12583
12584	case VMX_EXIT_INIT_SIGNAL:
12585	case VMX_EXIT_SIPI:
12586	case VMX_EXIT_IO_SMI:
12587	case VMX_EXIT_SMI:
12588	case VMX_EXIT_ERR_MSR_LOAD:
12589	case VMX_EXIT_ERR_MACHINE_CHECK:
12590	case VMX_EXIT_PML_FULL:
12591	case VMX_EXIT_VIRTUALIZED_EOI:
12592	case VMX_EXIT_GDTR_IDTR_ACCESS:
12593	case VMX_EXIT_LDTR_TR_ACCESS:
12594	case VMX_EXIT_APIC_WRITE:
12595	case VMX_EXIT_RDRAND:
12596	case VMX_EXIT_RSM:
12597	case VMX_EXIT_VMFUNC:
12598	case VMX_EXIT_ENCLS:
12599	case VMX_EXIT_RDSEED:
12600	case VMX_EXIT_XSAVES:
12601	case VMX_EXIT_XRSTORS:
12602	case VMX_EXIT_UMWAIT:
12603	case VMX_EXIT_TPAUSE:
12604	default:
12605	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12606	}
12607	#undef VMEXIT_CALL_RET
12608	}
12609	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12610
12611
12612	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12613	/**
12614	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12615	*
12616	* @returns Strict VBox status code (i.e. informational status codes too).
12617	* @param pVCpu The cross context virtual CPU structure.
12618	* @param pVmxTransient The VMX-transient structure.
12619	*/
12620	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12621	{
12622	uint32_t const uExitReason = pVmxTransient->uExitReason;
12623	switch (uExitReason)
12624	{
12625	case VMX_EXIT_EPT_MISCONFIG: return hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
12626	case VMX_EXIT_EPT_VIOLATION: return hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
12627	case VMX_EXIT_IO_INSTR: return hmR0VmxExitIoInstrNested(pVCpu, pVmxTransient);
12628	case VMX_EXIT_HLT: return hmR0VmxExitHltNested(pVCpu, pVmxTransient);
12629	case VMX_EXIT_XCPT_OR_NMI: return hmR0VmxExitXcptOrNmiNested(pVCpu, pVmxTransient);
12630
12631	/*
12632	* We shouldn't direct host physical interrupts to the nested-guest.
12633	*/
12634	case VMX_EXIT_EXT_INT:
12635	return hmR0VmxExitExtInt(pVCpu, pVmxTransient);
12636
12637	/*
12638	* Instructions that cause VM-exits unconditionally.
12639	* - Provides VM-exit instruction length ONLY.
12640	*/
12641	case VMX_EXIT_CPUID:
12642	case VMX_EXIT_VMCALL:
12643	case VMX_EXIT_GETSEC:
12644	case VMX_EXIT_INVD:
12645	case VMX_EXIT_XSETBV:
12646	case VMX_EXIT_VMLAUNCH:
12647	case VMX_EXIT_VMRESUME:
12648	case VMX_EXIT_VMXOFF:
12649	return hmR0VmxExitInstrNested(pVCpu, pVmxTransient);
12650
12651	/*
12652	* Instructions that cause VM-exits unconditionally.
12653	* - Provides VM-exit instruction length.
12654	* - Provides VM-exit information.
12655	* - Optionally provides VM-exit qualification.
12656	*
12657	* Since VM-exit qualification is 0 for all VM-exits where it is not
12658	* applicable, reading and passing it to the guest should produce
12659	* defined behavior.
12660	*
12661	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
12662	*/
12663	case VMX_EXIT_INVEPT:
12664	case VMX_EXIT_INVVPID:
12665	case VMX_EXIT_VMCLEAR:
12666	case VMX_EXIT_VMPTRLD:
12667	case VMX_EXIT_VMPTRST:
12668	case VMX_EXIT_VMXON:
12669	return hmR0VmxExitInstrWithInfoNested(pVCpu, pVmxTransient);
12670
12671	case VMX_EXIT_RDTSC: return hmR0VmxExitRdtscNested(pVCpu, pVmxTransient);
12672	case VMX_EXIT_RDTSCP: return hmR0VmxExitRdtscpNested(pVCpu, pVmxTransient);
12673	case VMX_EXIT_RDMSR: return hmR0VmxExitRdmsrNested(pVCpu, pVmxTransient);
12674	case VMX_EXIT_WRMSR: return hmR0VmxExitWrmsrNested(pVCpu, pVmxTransient);
12675	case VMX_EXIT_INVLPG: return hmR0VmxExitInvlpgNested(pVCpu, pVmxTransient);
12676	case VMX_EXIT_INVPCID: return hmR0VmxExitInvpcidNested(pVCpu, pVmxTransient);
12677	case VMX_EXIT_TASK_SWITCH: return hmR0VmxExitTaskSwitchNested(pVCpu, pVmxTransient);
12678	case VMX_EXIT_WBINVD: return hmR0VmxExitWbinvdNested(pVCpu, pVmxTransient);
12679	case VMX_EXIT_MTF: return hmR0VmxExitMtfNested(pVCpu, pVmxTransient);
12680	case VMX_EXIT_APIC_ACCESS: return hmR0VmxExitApicAccessNested(pVCpu, pVmxTransient);
12681	case VMX_EXIT_MOV_CRX: return hmR0VmxExitMovCRxNested(pVCpu, pVmxTransient);
12682	case VMX_EXIT_INT_WINDOW: return hmR0VmxExitIntWindowNested(pVCpu, pVmxTransient);
12683	case VMX_EXIT_TPR_BELOW_THRESHOLD: return hmR0VmxExitTprBelowThresholdNested(pVCpu, pVmxTransient);
12684	case VMX_EXIT_MWAIT: return hmR0VmxExitMwaitNested(pVCpu, pVmxTransient);
12685	case VMX_EXIT_MONITOR: return hmR0VmxExitMonitorNested(pVCpu, pVmxTransient);
12686	case VMX_EXIT_PAUSE: return hmR0VmxExitPauseNested(pVCpu, pVmxTransient);
12687
12688	case VMX_EXIT_PREEMPT_TIMER:
12689	{
12690	/** @todo NSTVMX: Preempt timer. */
12691	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12692	}
12693
12694	case VMX_EXIT_MOV_DRX: return hmR0VmxExitMovDRxNested(pVCpu, pVmxTransient);
12695	case VMX_EXIT_RDPMC: return hmR0VmxExitRdpmcNested(pVCpu, pVmxTransient);
12696
12697	case VMX_EXIT_GDTR_IDTR_ACCESS:
12698	case VMX_EXIT_LDTR_TR_ACCESS: return hmR0VmxExitXdtrAccessNested(pVCpu, pVmxTransient);
12699
12700	case VMX_EXIT_VMREAD:
12701	case VMX_EXIT_VMWRITE: return hmR0VmxExitVmreadVmwriteNested(pVCpu, pVmxTransient);
12702
12703	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFaultNested(pVCpu, pVmxTransient);
12704	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindowNested(pVCpu, pVmxTransient);
12705
12706	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12707	{
12708	/** @todo NSTVMX: Invalid guest state. */
12709	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12710	}
12711
12712	case VMX_EXIT_INIT_SIGNAL:
12713	case VMX_EXIT_SIPI:
12714	case VMX_EXIT_IO_SMI:
12715	case VMX_EXIT_SMI:
12716	case VMX_EXIT_ERR_MSR_LOAD:
12717	case VMX_EXIT_ERR_MACHINE_CHECK:
12718	case VMX_EXIT_PML_FULL:
12719	case VMX_EXIT_VIRTUALIZED_EOI:
12720	case VMX_EXIT_APIC_WRITE:
12721	case VMX_EXIT_RDRAND:
12722	case VMX_EXIT_RSM:
12723	case VMX_EXIT_VMFUNC:
12724	case VMX_EXIT_ENCLS:
12725	case VMX_EXIT_RDSEED:
12726	case VMX_EXIT_XSAVES:
12727	case VMX_EXIT_XRSTORS:
12728	case VMX_EXIT_UMWAIT:
12729	case VMX_EXIT_TPAUSE:
12730	default:
12731	{
12732	/** @todo NSTVMX: implement me! */
12733	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12734	}
12735	}
12736	}
12737	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12738
12739
12740	#ifdef VBOX_STRICT
12741	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
12742	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
12743	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
12744
12745	# define HMVMX_ASSERT_PREEMPT_CPUID() \
12746	do { \
12747	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
12748	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
12749	} while (0)
12750
12751	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12752	do { \
12753	AssertPtr((a_pVCpu)); \
12754	AssertPtr((a_pVmxTransient)); \
12755	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
12756	Assert((a_pVmxTransient)->pVmcsInfo); \
12757	Assert(ASMIntAreEnabled()); \
12758	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12759	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
12760	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
12761	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12762	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
12763	HMVMX_ASSERT_PREEMPT_CPUID(); \
12764	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12765	} while (0)
12766
12767	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12768	do { \
12769	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
12770	Assert((a_pVmxTransient)->fIsNestedGuest); \
12771	} while (0)
12772
12773	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12774	do { \
12775	Log4Func(("\n")); \
12776	} while (0)
12777	#else
12778	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12779	do { \
12780	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12781	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
12782	} while (0)
12783
12784	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12785	do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
12786
12787	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
12788	#endif
12789
12790
12791	/**
12792	* Advances the guest RIP by the specified number of bytes.
12793	*
12794	* @param pVCpu The cross context virtual CPU structure.
12795	* @param cbInstr Number of bytes to advance the RIP by.
12796	*
12797	* @remarks No-long-jump zone!!!
12798	*/
12799	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
12800	{
12801	/* Advance the RIP. */
12802	pVCpu->cpum.GstCtx.rip += cbInstr;
12803	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12804
12805	/* Update interrupt inhibition. */
12806	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
12807	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
12808	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12809	}
12810
12811
12812	/**
12813	* Advances the guest RIP after reading it from the VMCS.
12814	*
12815	* @returns VBox status code, no informational status codes.
12816	* @param pVCpu The cross context virtual CPU structure.
12817	* @param pVmxTransient The VMX-transient structure.
12818	*
12819	* @remarks No-long-jump zone!!!
12820	*/
12821	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12822	{
12823	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12824	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
12825	AssertRCReturn(rc, rc);
12826
12827	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
12828	return VINF_SUCCESS;
12829	}
12830
12831
12832	/**
12833	* Handle a condition that occurred while delivering an event through the guest
12834	* IDT.
12835	*
12836	* @returns Strict VBox status code (i.e. informational status codes too).
12837	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
12838	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
12839	* to continue execution of the guest which will delivery the \#DF.
12840	* @retval VINF_EM_RESET if we detected a triple-fault condition.
12841	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
12842	*
12843	* @param pVCpu The cross context virtual CPU structure.
12844	* @param pVmxTransient The VMX-transient structure.
12845	*
12846	* @remarks No-long-jump zone!!!
12847	*/
12848	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12849	{
12850	uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12851
12852	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12853	rc2 \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12854	AssertRCReturn(rc2, rc2);
12855
12856	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
12857	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12858	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12859	{
12860	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12861	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12862
12863	/*
12864	* If the event was a software interrupt (generated with INT n) or a software exception
12865	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
12866	* can handle the VM-exit and continue guest execution which will re-execute the
12867	* instruction rather than re-injecting the exception, as that can cause premature
12868	* trips to ring-3 before injection and involve TRPM which currently has no way of
12869	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
12870	* the problem).
12871	*/
12872	IEMXCPTRAISE enmRaise;
12873	IEMXCPTRAISEINFO fRaiseInfo;
12874	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
12875	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
12876	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
12877	{
12878	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
12879	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12880	}
12881	else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
12882	{
12883	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12884	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
12885	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
12886	/** @todo Make AssertMsgReturn as just AssertMsg later. */
12887	AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
12888	("Unexpected VM-exit interruption vector type %#x!\n", uExitVectorType), VERR_VMX_IPE_5);
12889
12890	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
12891
12892	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
12893	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
12894	{
12895	pVmxTransient->fVectoringPF = true;
12896	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12897	}
12898	}
12899	else
12900	{
12901	/*
12902	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
12903	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
12904	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
12905	*/
12906	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12907	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12908	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
12909	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12910	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12911	}
12912
12913	/*
12914	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
12915	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
12916	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
12917	* subsequent VM-entry would fail.
12918	*
12919	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
12920	*/
12921	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS)
12922	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12923	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
12924	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
12925	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
12926	{
12927	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
12928	}
12929
12930	switch (enmRaise)
12931	{
12932	case IEMXCPTRAISE_CURRENT_XCPT:
12933	{
12934	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
12935	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
12936	Assert(rcStrict == VINF_SUCCESS);
12937	break;
12938	}
12939
12940	case IEMXCPTRAISE_PREV_EVENT:
12941	{
12942	uint32_t u32ErrCode;
12943	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
12944	{
12945	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12946	AssertRCReturn(rc2, rc2);
12947	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12948	}
12949	else
12950	u32ErrCode = 0;
12951
12952	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
12953	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
12954	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12955	0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
12956
12957	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
12958	pVCpu->hm.s.Event.u32ErrCode));
12959	Assert(rcStrict == VINF_SUCCESS);
12960	break;
12961	}
12962
12963	case IEMXCPTRAISE_REEXEC_INSTR:
12964	Assert(rcStrict == VINF_SUCCESS);
12965	break;
12966
12967	case IEMXCPTRAISE_DOUBLE_FAULT:
12968	{
12969	/*
12970	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
12971	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
12972	*/
12973	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
12974	{
12975	pVmxTransient->fVectoringDoublePF = true;
12976	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
12977	pVCpu->cpum.GstCtx.cr2));
12978	rcStrict = VINF_SUCCESS;
12979	}
12980	else
12981	{
12982	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
12983	hmR0VmxSetPendingXcptDF(pVCpu);
12984	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
12985	uIdtVector, uExitVector));
12986	rcStrict = VINF_HM_DOUBLE_FAULT;
12987	}
12988	break;
12989	}
12990
12991	case IEMXCPTRAISE_TRIPLE_FAULT:
12992	{
12993	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
12994	rcStrict = VINF_EM_RESET;
12995	break;
12996	}
12997
12998	case IEMXCPTRAISE_CPU_HANG:
12999	{
13000	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13001	rcStrict = VERR_EM_GUEST_CPU_HANG;
13002	break;
13003	}
13004
13005	default:
13006	{
13007	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
13008	rcStrict = VERR_VMX_IPE_2;
13009	break;
13010	}
13011	}
13012	}
13013	else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
13014	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
13015	&& uExitVector != X86_XCPT_DF
13016	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
13017	{
13018	/*
13019	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
13020	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
13021	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
13022	*/
13023	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
13024	{
13025	Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
13026	VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
13027	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
13028	}
13029	}
13030
13031	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
13032	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
13033	return rcStrict;
13034	}
13035
13036
13037	/** @name VM-exit handlers.
13038	* @{
13039	*/
13040	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13041	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
13042	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13043
13044	/**
13045	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
13046	*/
13047	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13048	{
13049	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13050	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
13051	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
13052	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
13053	return VINF_SUCCESS;
13054	return VINF_EM_RAW_INTERRUPT;
13055	}
13056
13057
13058	/**
13059	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
13060	* VM-exit.
13061	*/
13062	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13063	{
13064	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13065	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
13066
13067	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13068	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13069	AssertRCReturn(rc, rc);
13070
13071	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
13072	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
13073	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
13074	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
13075
13076	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
13077	{
13078	/*
13079	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
13080	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
13081	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
13082	*
13083	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
13084	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
13085	*/
13086	return hmR0VmxExitHostNmi(pVCpu);
13087	}
13088
13089	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13090	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
13091	if (RT_UNLIKELY(rcStrict == VINF_SUCCESS))
13092	{ /* likely */ }
13093	else
13094	{
13095	if (rcStrict == VINF_HM_DOUBLE_FAULT)
13096	rcStrict = VINF_SUCCESS;
13097	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
13098	return rcStrict;
13099	}
13100
13101	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
13102	uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
13103	switch (uIntType)
13104	{
13105	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
13106	Assert(uVector == X86_XCPT_DB);
13107	RT_FALL_THRU();
13108	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
13109	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
13110	RT_FALL_THRU();
13111	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
13112	{
13113	/*
13114	* If there's any exception caused as a result of event injection, the resulting
13115	* secondary/final execption will be pending, we shall continue guest execution
13116	* after injecting the event. The page-fault case is complicated and we manually
13117	* handle any currently pending event in hmR0VmxExitXcptPF.
13118	*/
13119	if (!pVCpu->hm.s.Event.fPending)
13120	{ /* likely */ }
13121	else if (uVector != X86_XCPT_PF)
13122	{
13123	rcStrict = VINF_SUCCESS;
13124	break;
13125	}
13126
13127	switch (uVector)
13128	{
13129	case X86_XCPT_PF: rcStrict = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
13130	case X86_XCPT_GP: rcStrict = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
13131	case X86_XCPT_MF: rcStrict = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
13132	case X86_XCPT_DB: rcStrict = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
13133	case X86_XCPT_BP: rcStrict = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
13134	case X86_XCPT_AC: rcStrict = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
13135
13136	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13137	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13138	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
13139	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13140	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
13141	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13142	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
13143	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13144	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
13145	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13146	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
13147	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13148	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
13149	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13150	default:
13151	{
13152	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
13153	if (pVmcsInfo->RealMode.fRealOnV86Active)
13154	{
13155	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
13156	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
13157	Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
13158
13159	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
13160	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13161	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13162	AssertRCReturn(rc, rc);
13163	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
13164	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
13165	0 /* GCPtrFaultAddress */);
13166	rcStrict = VINF_SUCCESS;
13167	}
13168	else
13169	{
13170	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
13171	pVCpu->hm.s.u32HMError = uVector;
13172	rcStrict = VERR_VMX_UNEXPECTED_EXCEPTION;
13173	}
13174	break;
13175	}
13176	}
13177	break;
13178	}
13179
13180	default:
13181	{
13182	pVCpu->hm.s.u32HMError = uExitIntInfo;
13183	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
13184	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
13185	break;
13186	}
13187	}
13188	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
13189	return rcStrict;
13190	}
13191
13192
13193	/**
13194	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
13195	*/
13196	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13197	{
13198	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13199
13200	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
13201	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13202	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
13203	AssertRCReturn(rc, rc);
13204
13205	/* Evaluate and deliver pending events and resume guest execution. */
13206	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
13207	return VINF_SUCCESS;
13208	}
13209
13210
13211	/**
13212	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
13213	*/
13214	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13215	{
13216	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13217
13218	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13219	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
13220	{
13221	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
13222	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
13223	}
13224
13225	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS));
13226
13227	/*
13228	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
13229	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
13230	*/
13231	uint32_t fIntrState;
13232	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13233	AssertRCReturn(rc, rc);
13234	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
13235	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
13236	{
13237	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
13238	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
13239
13240	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
13241	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
13242	AssertRCReturn(rc, rc);
13243	}
13244
13245	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
13246	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
13247	AssertRCReturn(rc, rc);
13248
13249	/* Evaluate and deliver pending events and resume guest execution. */
13250	return VINF_SUCCESS;
13251	}
13252
13253
13254	/**
13255	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
13256	*/
13257	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13258	{
13259	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13260	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13261	}
13262
13263
13264	/**
13265	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
13266	*/
13267	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13268	{
13269	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13270	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13271	}
13272
13273
13274	/**
13275	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
13276	*/
13277	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13278	{
13279	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13280
13281	/*
13282	* Get the state we need and update the exit history entry.
13283	*/
13284	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13285	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13286	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13287	AssertRCReturn(rc, rc);
13288
13289	VBOXSTRICTRC rcStrict;
13290	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
13291	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
13292	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
13293	if (!pExitRec)
13294	{
13295	/*
13296	* Regular CPUID instruction execution.
13297	*/
13298	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
13299	if (rcStrict == VINF_SUCCESS)
13300	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13301	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13302	{
13303	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13304	rcStrict = VINF_SUCCESS;
13305	}
13306	}
13307	else
13308	{
13309	/*
13310	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
13311	*/
13312	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13313	AssertRCReturn(rc2, rc2);
13314
13315	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
13316	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
13317
13318	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
13319	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13320
13321	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
13322	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13323	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
13324	}
13325	return rcStrict;
13326	}
13327
13328
13329	/**
13330	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
13331	*/
13332	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13333	{
13334	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13335
13336	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13337	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
13338	AssertRCReturn(rc, rc);
13339
13340	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
13341	return VINF_EM_RAW_EMULATE_INSTR;
13342
13343	AssertMsgFailed(("hmR0VmxExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
13344	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
13345	}
13346
13347
13348	/**
13349	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
13350	*/
13351	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13352	{
13353	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13354
13355	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13356	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13357	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13358	AssertRCReturn(rc, rc);
13359
13360	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
13361	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13362	{
13363	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13364	we must reset offsetting on VM-entry. See @bugref{6634}. */
13365	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13366	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13367	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13368	}
13369	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13370	{
13371	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13372	rcStrict = VINF_SUCCESS;
13373	}
13374	return rcStrict;
13375	}
13376
13377
13378	/**
13379	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
13380	*/
13381	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13382	{
13383	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13384
13385	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13386	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
13387	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13388	AssertRCReturn(rc, rc);
13389
13390	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
13391	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13392	{
13393	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13394	we must reset offsetting on VM-reentry. See @bugref{6634}. */
13395	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13396	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13397	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13398	}
13399	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13400	{
13401	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13402	rcStrict = VINF_SUCCESS;
13403	}
13404	return rcStrict;
13405	}
13406
13407
13408	/**
13409	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
13410	*/
13411	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13412	{
13413	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13414
13415	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13416	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
13417	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13418	AssertRCReturn(rc, rc);
13419
13420	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13421	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13422	if (RT_LIKELY(rc == VINF_SUCCESS))
13423	{
13424	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13425	Assert(pVmxTransient->cbInstr == 2);
13426	}
13427	else
13428	{
13429	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
13430	rc = VERR_EM_INTERPRETER;
13431	}
13432	return rc;
13433	}
13434
13435
13436	/**
13437	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
13438	*/
13439	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13440	{
13441	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13442
13443	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
13444	if (EMAreHypercallInstructionsEnabled(pVCpu))
13445	{
13446	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13447	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
13448	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13449	AssertRCReturn(rc, rc);
13450
13451	/* Perform the hypercall. */
13452	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
13453	if (rcStrict == VINF_SUCCESS)
13454	{
13455	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13456	AssertRCReturn(rc, rc);
13457	}
13458	else
13459	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
13460	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
13461	\|\| RT_FAILURE(rcStrict));
13462
13463	/* If the hypercall changes anything other than guest's general-purpose registers,
13464	we would need to reload the guest changed bits here before VM-entry. */
13465	}
13466	else
13467	Log4Func(("Hypercalls not enabled\n"));
13468
13469	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
13470	if (RT_FAILURE(rcStrict))
13471	{
13472	hmR0VmxSetPendingXcptUD(pVCpu);
13473	rcStrict = VINF_SUCCESS;
13474	}
13475
13476	return rcStrict;
13477	}
13478
13479
13480	/**
13481	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
13482	*/
13483	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13484	{
13485	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13486	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
13487
13488	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13489	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13490	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13491	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13492	AssertRCReturn(rc, rc);
13493
13494	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
13495
13496	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
13497	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13498	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13499	{
13500	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13501	rcStrict = VINF_SUCCESS;
13502	}
13503	else
13504	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
13505	VBOXSTRICTRC_VAL(rcStrict)));
13506	return rcStrict;
13507	}
13508
13509
13510	/**
13511	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
13512	*/
13513	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13514	{
13515	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13516
13517	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13518	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
13519	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13520	AssertRCReturn(rc, rc);
13521
13522	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbInstr);
13523	if (rcStrict == VINF_SUCCESS)
13524	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13525	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13526	{
13527	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13528	rcStrict = VINF_SUCCESS;
13529	}
13530
13531	return rcStrict;
13532	}
13533
13534
13535	/**
13536	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
13537	*/
13538	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13539	{
13540	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13541
13542	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13543	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13544	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13545	AssertRCReturn(rc, rc);
13546
13547	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbInstr);
13548	if (RT_SUCCESS(rcStrict))
13549	{
13550	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13551	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
13552	rcStrict = VINF_SUCCESS;
13553	}
13554
13555	return rcStrict;
13556	}
13557
13558
13559	/**
13560	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
13561	* VM-exit.
13562	*/
13563	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13564	{
13565	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13566	return VINF_EM_RESET;
13567	}
13568
13569
13570	/**
13571	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
13572	*/
13573	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13574	{
13575	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13576
13577	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13578	AssertRCReturn(rc, rc);
13579
13580	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
13581	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
13582	rc = VINF_SUCCESS;
13583	else
13584	rc = VINF_EM_HALT;
13585
13586	if (rc != VINF_SUCCESS)
13587	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
13588	return rc;
13589	}
13590
13591
13592	/**
13593	* VM-exit handler for instructions that result in a \#UD exception delivered to
13594	* the guest.
13595	*/
13596	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13597	{
13598	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13599	hmR0VmxSetPendingXcptUD(pVCpu);
13600	return VINF_SUCCESS;
13601	}
13602
13603
13604	/**
13605	* VM-exit handler for expiry of the VMX-preemption timer.
13606	*/
13607	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13608	{
13609	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13610
13611	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
13612	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13613
13614	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
13615	PVM pVM = pVCpu->CTX_SUFF(pVM);
13616	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
13617	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
13618	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
13619	}
13620
13621
13622	/**
13623	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
13624	*/
13625	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13626	{
13627	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13628
13629	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13630	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13631	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
13632	AssertRCReturn(rc, rc);
13633
13634	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
13635	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13636	: HM_CHANGED_RAISED_XCPT_MASK);
13637
13638	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13639	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
13640
13641	return rcStrict;
13642	}
13643
13644
13645	/**
13646	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
13647	*/
13648	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13649	{
13650	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13651	/** @todo Use VM-exit instruction information. */
13652	return VERR_EM_INTERPRETER;
13653	}
13654
13655
13656	/**
13657	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
13658	* Error VM-exit.
13659	*/
13660	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13661	{
13662	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13663	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13664	AssertRCReturn(rc, rc);
13665
13666	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo);
13667	if (RT_FAILURE(rc))
13668	return rc;
13669
13670	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
13671	NOREF(uInvalidReason);
13672
13673	#ifdef VBOX_STRICT
13674	uint32_t fIntrState;
13675	RTHCUINTREG uHCReg;
13676	uint64_t u64Val;
13677	uint32_t u32Val;
13678	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
13679	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
13680	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
13681	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13682	AssertRCReturn(rc, rc);
13683
13684	Log4(("uInvalidReason %u\n", uInvalidReason));
13685	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
13686	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
13687	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
13688	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
13689
13690	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
13691	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
13692	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
13693	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
13694	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
13695	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13696	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
13697	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
13698	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
13699	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13700	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
13701	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
13702
13703	hmR0DumpRegs(pVCpu);
13704	#endif
13705
13706	return VERR_VMX_INVALID_GUEST_STATE;
13707	}
13708
13709	/**
13710	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
13711	*/
13712	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUnexpected(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13713	{
13714	/*
13715	* Cummulative notes of all recognized but unexpected VM-exits.
13716	*
13717	* 1. This does -not- cover scenarios like like a page-fault VM-exit occurring when
13718	* nested-paging is used.
13719	*
13720	* 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
13721	* emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
13722	* this function (and thereby stop VM execution) for handling such instructions.
13723	*
13724	*
13725	* VMX_EXIT_INIT_SIGNAL:
13726	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
13727	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
13728	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
13729	*
13730	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
13731	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
13732	* See Intel spec. "23.8 Restrictions on VMX operation".
13733	*
13734	* VMX_EXIT_SIPI:
13735	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
13736	* activity state is used. We don't make use of it as our guests don't have direct
13737	* access to the host local APIC.
13738	*
13739	* See Intel spec. 25.3 "Other Causes of VM-exits".
13740	*
13741	* VMX_EXIT_IO_SMI:
13742	* VMX_EXIT_SMI:
13743	* This can only happen if we support dual-monitor treatment of SMI, which can be
13744	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
13745	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
13746	* VMX root mode or receive an SMI. If we get here, something funny is going on.
13747	*
13748	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
13749	* See Intel spec. 25.3 "Other Causes of VM-Exits"
13750	*
13751	* VMX_EXIT_ERR_MSR_LOAD:
13752	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
13753	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
13754	* execution.
13755	*
13756	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
13757	*
13758	* VMX_EXIT_ERR_MACHINE_CHECK:
13759	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
13760	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
13761	* #MC exception abort class exception is raised. We thus cannot assume a
13762	* reasonable chance of continuing any sort of execution and we bail.
13763	*
13764	* See Intel spec. 15.1 "Machine-check Architecture".
13765	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
13766	*
13767	* VMX_EXIT_PML_FULL:
13768	* VMX_EXIT_VIRTUALIZED_EOI:
13769	* VMX_EXIT_APIC_WRITE:
13770	* We do not currently support any of these features and thus they are all unexpected
13771	* VM-exits.
13772	*
13773	* VMX_EXIT_GDTR_IDTR_ACCESS:
13774	* VMX_EXIT_LDTR_TR_ACCESS:
13775	* VMX_EXIT_RDRAND:
13776	* VMX_EXIT_RSM:
13777	* VMX_EXIT_VMFUNC:
13778	* VMX_EXIT_ENCLS:
13779	* VMX_EXIT_RDSEED:
13780	* VMX_EXIT_XSAVES:
13781	* VMX_EXIT_XRSTORS:
13782	* VMX_EXIT_UMWAIT:
13783	* VMX_EXIT_TPAUSE:
13784	* These VM-exits are -not- caused unconditionally by execution of the corresponding
13785	* instruction. Any VM-exit for these instructions indicate a hardware problem,
13786	* unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
13787	*
13788	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
13789	*/
13790	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13791	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
13792	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
13793	}
13794
13795
13796	/**
13797	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
13798	*/
13799	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13800	{
13801	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13802
13803	/** @todo Optimize this: We currently drag in in the whole MSR state
13804	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13805	* MSRs required. That would require changes to IEM and possibly CPUM too.
13806	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13807	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13808	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13809	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13810	switch (idMsr)
13811	{
13812	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13813	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13814	}
13815
13816	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13817	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13818	AssertRCReturn(rc, rc);
13819
13820	Log4Func(("ecx=%#RX32\n", idMsr));
13821
13822	#ifdef VBOX_STRICT
13823	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
13824	{
13825	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
13826	&& idMsr != MSR_K6_EFER)
13827	{
13828	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
13829	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
13830	}
13831	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13832	{
13833	Assert(pVmcsInfo->pvMsrBitmap);
13834	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13835	if (fMsrpm & VMXMSRPM_ALLOW_RD)
13836	{
13837	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
13838	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
13839	}
13840	}
13841	}
13842	#endif
13843
13844	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
13845	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
13846	if (rcStrict == VINF_SUCCESS)
13847	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13848	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
13849	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13850	{
13851	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13852	rcStrict = VINF_SUCCESS;
13853	}
13854	else
13855	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13856
13857	return rcStrict;
13858	}
13859
13860
13861	/**
13862	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
13863	*/
13864	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13865	{
13866	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13867
13868	/** @todo Optimize this: We currently drag in in the whole MSR state
13869	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13870	* MSRs required. That would require changes to IEM and possibly CPUM too.
13871	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13872	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13873	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13874
13875	/*
13876	* The FS and GS base MSRs are not part of the above all-MSRs mask.
13877	* Although we don't need to fetch the base as it will be overwritten shortly, while
13878	* loading guest-state we would also load the entire segment register including limit
13879	* and attributes and thus we need to load them here.
13880	*/
13881	switch (idMsr)
13882	{
13883	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13884	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13885	}
13886
13887	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13888	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13889	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13890	AssertRCReturn(rc, rc);
13891
13892	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
13893
13894	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
13895	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
13896
13897	if (rcStrict == VINF_SUCCESS)
13898	{
13899	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13900
13901	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
13902	if ( idMsr == MSR_IA32_APICBASE
13903	\|\| ( idMsr >= MSR_IA32_X2APIC_START
13904	&& idMsr <= MSR_IA32_X2APIC_END))
13905	{
13906	/*
13907	* We've already saved the APIC related guest-state (TPR) in post-run phase.
13908	* When full APIC register virtualization is implemented we'll have to make
13909	* sure APIC state is saved from the VMCS before IEM changes it.
13910	*/
13911	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
13912	}
13913	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
13914	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13915	else if (idMsr == MSR_K6_EFER)
13916	{
13917	/*
13918	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
13919	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
13920	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
13921	*/
13922	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
13923	}
13924
13925	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
13926	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
13927	{
13928	switch (idMsr)
13929	{
13930	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
13931	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
13932	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
13933	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
13934	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
13935	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
13936	default:
13937	{
13938	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
13939	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
13940	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13941	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
13942	break;
13943	}
13944	}
13945	}
13946	#ifdef VBOX_STRICT
13947	else
13948	{
13949	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
13950	switch (idMsr)
13951	{
13952	case MSR_IA32_SYSENTER_CS:
13953	case MSR_IA32_SYSENTER_EIP:
13954	case MSR_IA32_SYSENTER_ESP:
13955	case MSR_K8_FS_BASE:
13956	case MSR_K8_GS_BASE:
13957	{
13958	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
13959	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
13960	}
13961
13962	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
13963	default:
13964	{
13965	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
13966	{
13967	/* EFER MSR writes are always intercepted. */
13968	if (idMsr != MSR_K6_EFER)
13969	{
13970	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
13971	idMsr));
13972	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
13973	}
13974	}
13975
13976	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13977	{
13978	Assert(pVmcsInfo->pvMsrBitmap);
13979	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13980	if (fMsrpm & VMXMSRPM_ALLOW_WR)
13981	{
13982	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
13983	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
13984	}
13985	}
13986	break;
13987	}
13988	}
13989	}
13990	#endif /* VBOX_STRICT */
13991	}
13992	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13993	{
13994	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13995	rcStrict = VINF_SUCCESS;
13996	}
13997	else
13998	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13999
14000	return rcStrict;
14001	}
14002
14003
14004	/**
14005	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
14006	*/
14007	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14008	{
14009	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14010
14011	/** @todo The guest has likely hit a contended spinlock. We might want to
14012	* poke a schedule different guest VCPU. */
14013	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14014	if (RT_SUCCESS(rc))
14015	return VINF_EM_RAW_INTERRUPT;
14016
14017	AssertMsgFailed(("hmR0VmxExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
14018	return rc;
14019	}
14020
14021
14022	/**
14023	* VM-exit handler for when the TPR value is lowered below the specified
14024	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
14025	*/
14026	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14027	{
14028	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14029	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
14030
14031	/*
14032	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
14033	* We'll re-evaluate pending interrupts and inject them before the next VM
14034	* entry so we can just continue execution here.
14035	*/
14036	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
14037	return VINF_SUCCESS;
14038	}
14039
14040
14041	/**
14042	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
14043	* VM-exit.
14044	*
14045	* @retval VINF_SUCCESS when guest execution can continue.
14046	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
14047	* @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
14048	* incompatible guest state for VMX execution (real-on-v86 case).
14049	*/
14050	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14051	{
14052	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14053	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
14054
14055	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14056	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14057	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14058	AssertRCReturn(rc, rc);
14059
14060	VBOXSTRICTRC rcStrict;
14061	PVM pVM = pVCpu->CTX_SUFF(pVM);
14062	uint64_t const uExitQual = pVmxTransient->uExitQual;
14063	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
14064	switch (uAccessType)
14065	{
14066	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
14067	{
14068	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14069	AssertRCReturn(rc, rc);
14070
14071	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
14072	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
14073	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
14074	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
14075
14076	/*
14077	* MOV to CR3 only cause a VM-exit when one or more of the following are true:
14078	* - When nested paging isn't used.
14079	* - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
14080	* - We are executing in the VM debug loop.
14081	*/
14082	Assert( iCrReg != 3
14083	\|\| !pVM->hm.s.fNestedPaging
14084	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
14085	\|\| pVCpu->hm.s.fUsingDebugLoop);
14086
14087	/* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
14088	Assert( iCrReg != 8
14089	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
14090
14091	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
14092	AssertMsg( rcStrict == VINF_SUCCESS
14093	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14094
14095	/*
14096	* This is a kludge for handling switches back to real mode when we try to use
14097	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
14098	* deal with special selector values, so we have to return to ring-3 and run
14099	* there till the selector values are V86 mode compatible.
14100	*
14101	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
14102	* latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
14103	* this function.
14104	*/
14105	if ( iCrReg == 0
14106	&& rcStrict == VINF_SUCCESS
14107	&& !pVM->hm.s.vmx.fUnrestrictedGuest
14108	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
14109	&& (uOldCr0 & X86_CR0_PE)
14110	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
14111	{
14112	/** @todo Check selectors rather than returning all the time. */
14113	Assert(!pVmxTransient->fIsNestedGuest);
14114	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
14115	rcStrict = VINF_EM_RESCHEDULE_REM;
14116	}
14117	break;
14118	}
14119
14120	case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
14121	{
14122	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
14123	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
14124
14125	/*
14126	* MOV from CR3 only cause a VM-exit when one or more of the following are true:
14127	* - When nested paging isn't used.
14128	* - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
14129	* - We are executing in the VM debug loop.
14130	*/
14131	Assert( iCrReg != 3
14132	\|\| !pVM->hm.s.fNestedPaging
14133	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
14134	\|\| pVCpu->hm.s.fUsingDebugLoop);
14135
14136	/* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
14137	Assert( iCrReg != 8
14138	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
14139
14140	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
14141	break;
14142	}
14143
14144	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
14145	{
14146	/*
14147	* CLTS (Clear Task-Switch Flag in CR0).
14148	*/
14149	rcStrict = hmR0VmxExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbInstr);
14150	break;
14151	}
14152
14153	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
14154	{
14155	/*
14156	* LMSW (Load Machine-Status Word into CR0).
14157	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
14158	*/
14159	RTGCPTR GCPtrEffDst;
14160	uint8_t const cbInstr = pVmxTransient->cbInstr;
14161	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
14162	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
14163	if (fMemOperand)
14164	{
14165	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
14166	AssertRCReturn(rc, rc);
14167	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
14168	}
14169	else
14170	GCPtrEffDst = NIL_RTGCPTR;
14171	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
14172	break;
14173	}
14174
14175	default:
14176	{
14177	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
14178	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
14179	}
14180	}
14181
14182	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
14183	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
14184	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
14185
14186	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
14187	NOREF(pVM);
14188	return rcStrict;
14189	}
14190
14191
14192	/**
14193	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
14194	* VM-exit.
14195	*/
14196	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14197	{
14198	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14199	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
14200
14201	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14202	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14203	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14204	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14205	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
14206	\| CPUMCTX_EXTRN_EFER);
14207	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
14208	AssertRCReturn(rc, rc);
14209
14210	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
14211	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
14212	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
14213	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
14214	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
14215	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
14216	bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
14217	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
14218
14219	/*
14220	* Update exit history to see if this exit can be optimized.
14221	*/
14222	VBOXSTRICTRC rcStrict;
14223	PCEMEXITREC pExitRec = NULL;
14224	if ( !fGstStepping
14225	&& !fDbgStepping)
14226	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14227	!fIOString
14228	? !fIOWrite
14229	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
14230	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
14231	: !fIOWrite
14232	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
14233	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
14234	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14235	if (!pExitRec)
14236	{
14237	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
14238	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
14239
14240	uint32_t const cbValue = s_aIOSizes[uIOSize];
14241	uint32_t const cbInstr = pVmxTransient->cbInstr;
14242	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
14243	PVM pVM = pVCpu->CTX_SUFF(pVM);
14244	if (fIOString)
14245	{
14246	/*
14247	* INS/OUTS - I/O String instruction.
14248	*
14249	* Use instruction-information if available, otherwise fall back on
14250	* interpreting the instruction.
14251	*/
14252	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14253	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
14254	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
14255	if (fInsOutsInfo)
14256	{
14257	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14258	AssertRCReturn(rc2, rc2);
14259	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
14260	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
14261	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
14262	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
14263	if (fIOWrite)
14264	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
14265	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
14266	else
14267	{
14268	/*
14269	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
14270	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
14271	* See Intel Instruction spec. for "INS".
14272	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
14273	*/
14274	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
14275	}
14276	}
14277	else
14278	rcStrict = IEMExecOne(pVCpu);
14279
14280	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14281	fUpdateRipAlready = true;
14282	}
14283	else
14284	{
14285	/*
14286	* IN/OUT - I/O instruction.
14287	*/
14288	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14289	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
14290	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
14291	if (fIOWrite)
14292	{
14293	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
14294	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
14295	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14296	&& !pCtx->eflags.Bits.u1TF)
14297	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
14298	}
14299	else
14300	{
14301	uint32_t u32Result = 0;
14302	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
14303	if (IOM_SUCCESS(rcStrict))
14304	{
14305	/* Save result of I/O IN instr. in AL/AX/EAX. */
14306	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
14307	}
14308	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14309	&& !pCtx->eflags.Bits.u1TF)
14310	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
14311	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
14312	}
14313	}
14314
14315	if (IOM_SUCCESS(rcStrict))
14316	{
14317	if (!fUpdateRipAlready)
14318	{
14319	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
14320	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14321	}
14322
14323	/*
14324	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
14325	* while booting Fedora 17 64-bit guest.
14326	*
14327	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
14328	*/
14329	if (fIOString)
14330	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
14331
14332	/*
14333	* If any I/O breakpoints are armed, we need to check if one triggered
14334	* and take appropriate action.
14335	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
14336	*/
14337	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
14338	AssertRCReturn(rc, rc);
14339
14340	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
14341	* execution engines about whether hyper BPs and such are pending. */
14342	uint32_t const uDr7 = pCtx->dr[7];
14343	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
14344	&& X86_DR7_ANY_RW_IO(uDr7)
14345	&& (pCtx->cr4 & X86_CR4_DE))
14346	\|\| DBGFBpIsHwIoArmed(pVM)))
14347	{
14348	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
14349
14350	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
14351	VMMRZCallRing3Disable(pVCpu);
14352	HM_DISABLE_PREEMPT(pVCpu);
14353
14354	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
14355
14356	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
14357	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
14358	{
14359	/* Raise #DB. */
14360	if (fIsGuestDbgActive)
14361	ASMSetDR6(pCtx->dr[6]);
14362	if (pCtx->dr[7] != uDr7)
14363	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
14364
14365	hmR0VmxSetPendingXcptDB(pVCpu);
14366	}
14367	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
14368	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
14369	else if ( rcStrict2 != VINF_SUCCESS
14370	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
14371	rcStrict = rcStrict2;
14372	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
14373
14374	HM_RESTORE_PREEMPT();
14375	VMMRZCallRing3Enable(pVCpu);
14376	}
14377	}
14378
14379	#ifdef VBOX_STRICT
14380	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14381	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
14382	Assert(!fIOWrite);
14383	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14384	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
14385	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
14386	Assert(fIOWrite);
14387	else
14388	{
14389	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
14390	* statuses, that the VMM device and some others may return. See
14391	* IOM_SUCCESS() for guidance. */
14392	AssertMsg( RT_FAILURE(rcStrict)
14393	\|\| rcStrict == VINF_SUCCESS
14394	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
14395	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
14396	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
14397	\|\| rcStrict == VINF_EM_RAW_TO_R3
14398	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14399	# endif
14400	}
14401	#endif
14402	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
14403	}
14404	else
14405	{
14406	/*
14407	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14408	*/
14409	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14410	AssertRCReturn(rc2, rc2);
14411	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
14412	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
14413	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
14414	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14415	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
14416	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
14417
14418	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14419	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14420
14421	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14422	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14423	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14424	}
14425	return rcStrict;
14426	}
14427
14428
14429	/**
14430	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
14431	* VM-exit.
14432	*/
14433	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14434	{
14435	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14436
14437	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
14438	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14439	AssertRCReturn(rc, rc);
14440	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
14441	{
14442	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14443	AssertRCReturn(rc, rc);
14444	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
14445	{
14446	uint32_t uErrCode;
14447	RTGCUINTPTR GCPtrFaultAddress;
14448	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
14449	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
14450	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
14451	if (fErrorCodeValid)
14452	{
14453	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14454	AssertRCReturn(rc, rc);
14455	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
14456	}
14457	else
14458	uErrCode = 0;
14459
14460	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
14461	&& uVector == X86_XCPT_PF)
14462	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
14463	else
14464	GCPtrFaultAddress = 0;
14465
14466	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14467	AssertRCReturn(rc, rc);
14468
14469	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
14470	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
14471
14472	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
14473	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14474	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14475	}
14476	}
14477
14478	/* Fall back to the interpreter to emulate the task-switch. */
14479	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14480	return VERR_EM_INTERPRETER;
14481	}
14482
14483
14484	/**
14485	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
14486	*/
14487	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14488	{
14489	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14490
14491	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14492	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
14493	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14494	AssertRCReturn(rc, rc);
14495	return VINF_EM_DBG_STEPPED;
14496	}
14497
14498
14499	/**
14500	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
14501	*/
14502	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14503	{
14504	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14505	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
14506
14507	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14508	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14509	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14510	{
14511	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
14512	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14513	{
14514	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14515	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14516	}
14517	}
14518	else
14519	{
14520	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14521	rcStrict1 = VINF_SUCCESS;
14522	return rcStrict1;
14523	}
14524
14525	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
14526	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14527	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14528	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14529	AssertRCReturn(rc, rc);
14530
14531	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
14532	uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
14533	VBOXSTRICTRC rcStrict2;
14534	switch (uAccessType)
14535	{
14536	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
14537	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
14538	{
14539	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
14540	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
14541	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
14542
14543	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
14544	GCPhys &= PAGE_BASE_GC_MASK;
14545	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
14546	PVM pVM = pVCpu->CTX_SUFF(pVM);
14547	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
14548	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
14549
14550	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14551	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
14552	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
14553	CPUMCTX2CORE(pCtx), GCPhys);
14554	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14555	if ( rcStrict2 == VINF_SUCCESS
14556	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14557	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14558	{
14559	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14560	\| HM_CHANGED_GUEST_APIC_TPR);
14561	rcStrict2 = VINF_SUCCESS;
14562	}
14563	break;
14564	}
14565
14566	default:
14567	Log4Func(("uAccessType=%#x\n", uAccessType));
14568	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
14569	break;
14570	}
14571
14572	if (rcStrict2 != VINF_SUCCESS)
14573	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
14574	return rcStrict2;
14575	}
14576
14577
14578	/**
14579	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
14580	* VM-exit.
14581	*/
14582	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14583	{
14584	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14585
14586	/* We should -not- get this VM-exit if the guest's debug registers were active. */
14587	if (pVmxTransient->fWasGuestDebugStateActive)
14588	{
14589	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
14590	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14591	}
14592
14593	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14594	if ( !pVCpu->hm.s.fSingleInstruction
14595	&& !pVmxTransient->fWasHyperDebugStateActive)
14596	{
14597	Assert(!DBGFIsStepping(pVCpu));
14598	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
14599
14600	/* Don't intercept MOV DRx any more. */
14601	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
14602	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14603	AssertRCReturn(rc, rc);
14604
14605	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
14606	VMMRZCallRing3Disable(pVCpu);
14607	HM_DISABLE_PREEMPT(pVCpu);
14608
14609	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
14610	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
14611	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
14612
14613	HM_RESTORE_PREEMPT();
14614	VMMRZCallRing3Enable(pVCpu);
14615
14616	#ifdef VBOX_WITH_STATISTICS
14617	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14618	AssertRCReturn(rc, rc);
14619	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14620	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14621	else
14622	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14623	#endif
14624	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
14625	return VINF_SUCCESS;
14626	}
14627
14628	/*
14629	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
14630	* The EFER MSR is always up-to-date.
14631	* Update the segment registers and DR7 from the CPU.
14632	*/
14633	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14634	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14635	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
14636	AssertRCReturn(rc, rc);
14637	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
14638
14639	PVM pVM = pVCpu->CTX_SUFF(pVM);
14640	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14641	{
14642	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14643	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
14644	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
14645	if (RT_SUCCESS(rc))
14646	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
14647	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14648	}
14649	else
14650	{
14651	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14652	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
14653	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
14654	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14655	}
14656
14657	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
14658	if (RT_SUCCESS(rc))
14659	{
14660	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14661	AssertRCReturn(rc2, rc2);
14662	return VINF_SUCCESS;
14663	}
14664	return rc;
14665	}
14666
14667
14668	/**
14669	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
14670	* Conditional VM-exit.
14671	*/
14672	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14673	{
14674	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14675	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14676
14677	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14678	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14679	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14680	{
14681	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
14682	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
14683	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14684	{
14685	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14686	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14687	}
14688	}
14689	else
14690	{
14691	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14692	rcStrict1 = VINF_SUCCESS;
14693	return rcStrict1;
14694	}
14695
14696	/*
14697	* Get sufficent state and update the exit history entry.
14698	*/
14699	RTGCPHYS GCPhys;
14700	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14701	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14702	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14703	AssertRCReturn(rc, rc);
14704
14705	VBOXSTRICTRC rcStrict;
14706	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14707	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
14708	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14709	if (!pExitRec)
14710	{
14711	/*
14712	* If we succeed, resume guest execution.
14713	* If we fail in interpreting the instruction because we couldn't get the guest physical address
14714	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
14715	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
14716	* weird case. See @bugref{6043}.
14717	*/
14718	PVM pVM = pVCpu->CTX_SUFF(pVM);
14719	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14720	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
14721	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
14722	if ( rcStrict == VINF_SUCCESS
14723	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
14724	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
14725	{
14726	/* Successfully handled MMIO operation. */
14727	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14728	\| HM_CHANGED_GUEST_APIC_TPR);
14729	rcStrict = VINF_SUCCESS;
14730	}
14731	}
14732	else
14733	{
14734	/*
14735	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14736	*/
14737	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14738	AssertRCReturn(rc2, rc2);
14739
14740	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
14741	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
14742
14743	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14744	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14745
14746	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14747	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14748	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14749	}
14750	return VBOXSTRICTRC_TODO(rcStrict);
14751	}
14752
14753
14754	/**
14755	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
14756	* VM-exit.
14757	*/
14758	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14759	{
14760	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14761	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14762
14763	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14764	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14765	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14766	{
14767	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
14768	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14769	Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
14770	}
14771	else
14772	{
14773	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14774	rcStrict1 = VINF_SUCCESS;
14775	return rcStrict1;
14776	}
14777
14778	RTGCPHYS GCPhys;
14779	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14780	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14781	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14782	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14783	AssertRCReturn(rc, rc);
14784
14785	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
14786	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
14787
14788	RTGCUINT uErrorCode = 0;
14789	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
14790	uErrorCode \|= X86_TRAP_PF_ID;
14791	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
14792	uErrorCode \|= X86_TRAP_PF_RW;
14793	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
14794	uErrorCode \|= X86_TRAP_PF_P;
14795
14796	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
14797
14798
14799	/* Handle the pagefault trap for the nested shadow table. */
14800	PVM pVM = pVCpu->CTX_SUFF(pVM);
14801	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14802
14803	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
14804	pCtx->cs.Sel, pCtx->rip));
14805
14806	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
14807	TRPMResetTrap(pVCpu);
14808
14809	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
14810	if ( rcStrict2 == VINF_SUCCESS
14811	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14812	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14813	{
14814	/* Successfully synced our nested page tables. */
14815	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
14816	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
14817	return VINF_SUCCESS;
14818	}
14819
14820	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14821	return rcStrict2;
14822	}
14823
14824	/** @} */
14825
14826	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14827	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14828	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14829
14830	/**
14831	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
14832	*/
14833	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14834	{
14835	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14836	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
14837
14838	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
14839	AssertRCReturn(rc, rc);
14840
14841	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
14842	{
14843	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
14844	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
14845
14846	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
14847	* provides VM-exit instruction length. If this causes problem later,
14848	* disassemble the instruction like it's done on AMD-V. */
14849	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14850	AssertRCReturn(rc2, rc2);
14851	return rc;
14852	}
14853
14854	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14855	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14856	return rc;
14857	}
14858
14859
14860	/**
14861	* VM-exit exception handler for \#BP (Breakpoint exception).
14862	*/
14863	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14864	{
14865	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14866	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
14867
14868	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14869	AssertRCReturn(rc, rc);
14870
14871	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14872	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14873	if (rc == VINF_EM_RAW_GUEST_TRAP)
14874	{
14875	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14876	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14877	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14878	AssertRCReturn(rc, rc);
14879
14880	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14881	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14882	}
14883
14884	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
14885	return rc;
14886	}
14887
14888
14889	/**
14890	* VM-exit exception handler for \#AC (alignment check exception).
14891	*/
14892	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14893	{
14894	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14895
14896	/*
14897	* Re-inject it. We'll detect any nesting before getting here.
14898	*/
14899	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14900	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14901	AssertRCReturn(rc, rc);
14902	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
14903
14904	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14905	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14906	return VINF_SUCCESS;
14907	}
14908
14909
14910	/**
14911	* VM-exit exception handler for \#DB (Debug exception).
14912	*/
14913	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14914	{
14915	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14916	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
14917
14918	/*
14919	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
14920	* for processing.
14921	*/
14922	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14923
14924	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
14925	uint64_t const uDR6 = X86_DR6_INIT_VAL
14926	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
14927	\| X86_DR6_BD \| X86_DR6_BS));
14928
14929	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14930	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
14931	Log6Func(("rc=%Rrc\n", rc));
14932	if (rc == VINF_EM_RAW_GUEST_TRAP)
14933	{
14934	/*
14935	* The exception was for the guest. Update DR6, DR7.GD and
14936	* IA32_DEBUGCTL.LBR before forwarding it.
14937	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
14938	*/
14939	VMMRZCallRing3Disable(pVCpu);
14940	HM_DISABLE_PREEMPT(pVCpu);
14941
14942	pCtx->dr[6] &= ~X86_DR6_B_MASK;
14943	pCtx->dr[6] \|= uDR6;
14944	if (CPUMIsGuestDebugStateActive(pVCpu))
14945	ASMSetDR6(pCtx->dr[6]);
14946
14947	HM_RESTORE_PREEMPT();
14948	VMMRZCallRing3Enable(pVCpu);
14949
14950	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
14951	AssertRCReturn(rc, rc);
14952
14953	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
14954	pCtx->dr[7] &= ~X86_DR7_GD;
14955
14956	/* Paranoia. */
14957	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
14958	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
14959
14960	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
14961	AssertRCReturn(rc, rc);
14962
14963	/*
14964	* Raise #DB in the guest.
14965	*
14966	* It is important to reflect exactly what the VM-exit gave us (preserving the
14967	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
14968	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
14969	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
14970	*
14971	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
14972	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
14973	*/
14974	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14975	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14976	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14977	AssertRCReturn(rc, rc);
14978	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14979	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14980	return VINF_SUCCESS;
14981	}
14982
14983	/*
14984	* Not a guest trap, must be a hypervisor related debug event then.
14985	* Update DR6 in case someone is interested in it.
14986	*/
14987	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
14988	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
14989	CPUMSetHyperDR6(pVCpu, uDR6);
14990
14991	return rc;
14992	}
14993
14994
14995	/**
14996	* Hacks its way around the lovely mesa driver's backdoor accesses.
14997	*
14998	* @sa hmR0SvmHandleMesaDrvGp.
14999	*/
15000	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
15001	{
15002	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
15003	RT_NOREF(pCtx);
15004
15005	/* For now we'll just skip the instruction. */
15006	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15007	}
15008
15009
15010	/**
15011	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
15012	* backdoor logging w/o checking what it is running inside.
15013	*
15014	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
15015	* backdoor port and magic numbers loaded in registers.
15016	*
15017	* @returns true if it is, false if it isn't.
15018	* @sa hmR0SvmIsMesaDrvGp.
15019	*/
15020	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
15021	{
15022	/* 0xed: IN eAX,dx */
15023	uint8_t abInstr[1];
15024	if (pVmxTransient->cbInstr != sizeof(abInstr))
15025	return false;
15026
15027	/* Check that it is #GP(0). */
15028	if (pVmxTransient->uExitIntErrorCode != 0)
15029	return false;
15030
15031	/* Check magic and port. */
15032	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
15033	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
15034	if (pCtx->rax != UINT32_C(0x564d5868))
15035	return false;
15036	if (pCtx->dx != UINT32_C(0x5658))
15037	return false;
15038
15039	/* Flat ring-3 CS. */
15040	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
15041	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
15042	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
15043	if (pCtx->cs.Attr.n.u2Dpl != 3)
15044	return false;
15045	if (pCtx->cs.u64Base != 0)
15046	return false;
15047
15048	/* Check opcode. */
15049	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
15050	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
15051	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
15052	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
15053	if (RT_FAILURE(rc))
15054	return false;
15055	if (abInstr[0] != 0xed)
15056	return false;
15057
15058	return true;
15059	}
15060
15061
15062	/**
15063	* VM-exit exception handler for \#GP (General-protection exception).
15064	*
15065	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
15066	*/
15067	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15068	{
15069	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15070	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
15071
15072	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15073	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15074	if (pVmcsInfo->RealMode.fRealOnV86Active)
15075	{ /* likely */ }
15076	else
15077	{
15078	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
15079	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv);
15080	#endif
15081	/* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
15082	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15083	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15084	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15085	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15086	AssertRCReturn(rc, rc);
15087	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
15088	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
15089
15090	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
15091	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
15092	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
15093	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15094	else
15095	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
15096	return rc;
15097	}
15098
15099	Assert(CPUMIsGuestInRealModeEx(pCtx));
15100	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
15101
15102	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15103	AssertRCReturn(rc, rc);
15104
15105	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
15106	if (rcStrict == VINF_SUCCESS)
15107	{
15108	if (!CPUMIsGuestInRealModeEx(pCtx))
15109	{
15110	/*
15111	* The guest is no longer in real-mode, check if we can continue executing the
15112	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
15113	*/
15114	pVmcsInfo->RealMode.fRealOnV86Active = false;
15115	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
15116	{
15117	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
15118	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15119	}
15120	else
15121	{
15122	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
15123	rcStrict = VINF_EM_RESCHEDULE;
15124	}
15125	}
15126	else
15127	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15128	}
15129	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15130	{
15131	rcStrict = VINF_SUCCESS;
15132	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15133	}
15134	return VBOXSTRICTRC_VAL(rcStrict);
15135	}
15136
15137
15138	/**
15139	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
15140	* the exception reported in the VMX transient structure back into the VM.
15141	*
15142	* @remarks Requires uExitIntInfo in the VMX transient structure to be
15143	* up-to-date.
15144	*/
15145	static VBOXSTRICTRC hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15146	{
15147	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15148	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
15149	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15150	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active,
15151	("uVector=%#x u32XcptBitmap=%#X32\n",
15152	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
15153	NOREF(pVmcsInfo);
15154	#endif
15155
15156	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
15157	hmR0VmxCheckExitDueToEventDelivery(). */
15158	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15159	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15160	AssertRCReturn(rc, rc);
15161	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15162
15163	#ifdef DEBUG_ramshankar
15164	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
15165	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n",
15166	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pCtx->cs.Sel, pCtx->rip));
15167	#endif
15168
15169	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15170	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15171	return VINF_SUCCESS;
15172	}
15173
15174
15175	/**
15176	* VM-exit exception handler for \#PF (Page-fault exception).
15177	*/
15178	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15179	{
15180	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15181	PVM pVM = pVCpu->CTX_SUFF(pVM);
15182	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15183	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15184	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15185	AssertRCReturn(rc, rc);
15186
15187	if (!pVM->hm.s.fNestedPaging)
15188	{ /* likely */ }
15189	else
15190	{
15191	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
15192	Assert(pVCpu->hm.s.fUsingDebugLoop);
15193	#endif
15194	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
15195	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
15196	{
15197	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15198	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
15199	}
15200	else
15201	{
15202	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15203	hmR0VmxSetPendingXcptDF(pVCpu);
15204	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
15205	}
15206	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15207	return rc;
15208	}
15209
15210	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
15211	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
15212	if (pVmxTransient->fVectoringPF)
15213	{
15214	Assert(pVCpu->hm.s.Event.fPending);
15215	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15216	}
15217
15218	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15219	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15220	AssertRCReturn(rc, rc);
15221
15222	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
15223	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
15224
15225	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
15226	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
15227
15228	Log4Func(("#PF: rc=%Rrc\n", rc));
15229	if (rc == VINF_SUCCESS)
15230	{
15231	/*
15232	* This is typically a shadow page table sync or a MMIO instruction. But we may have
15233	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
15234	*/
15235	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15236	TRPMResetTrap(pVCpu);
15237	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
15238	return rc;
15239	}
15240
15241	if (rc == VINF_EM_RAW_GUEST_TRAP)
15242	{
15243	if (!pVmxTransient->fVectoringDoublePF)
15244	{
15245	/* It's a guest page fault and needs to be reflected to the guest. */
15246	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
15247	TRPMResetTrap(pVCpu);
15248	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
15249	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15250	uGstErrorCode, pVmxTransient->uExitQual);
15251	}
15252	else
15253	{
15254	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15255	TRPMResetTrap(pVCpu);
15256	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
15257	hmR0VmxSetPendingXcptDF(pVCpu);
15258	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
15259	}
15260
15261	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15262	return VINF_SUCCESS;
15263	}
15264
15265	TRPMResetTrap(pVCpu);
15266	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
15267	return rc;
15268	}
15269
15270
15271	/**
15272	* VM-exit helper for LMSW.
15273	*/
15274	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw,
15275	RTGCPTR GCPtrEffDst)
15276	{
15277	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15278	AssertRCReturn(rc, rc);
15279
15280	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
15281	AssertMsg( rcStrict == VINF_SUCCESS
15282	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15283
15284	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
15285	if (rcStrict == VINF_IEM_RAISED_XCPT)
15286	{
15287	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15288	rcStrict = VINF_SUCCESS;
15289	}
15290
15291	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
15292	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15293	return rcStrict;
15294	}
15295
15296
15297	/**
15298	* VM-exit helper for CLTS.
15299	*/
15300	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
15301	{
15302	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15303	AssertRCReturn(rc, rc);
15304
15305	VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
15306	AssertMsg( rcStrict == VINF_SUCCESS
15307	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15308
15309	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
15310	if (rcStrict == VINF_IEM_RAISED_XCPT)
15311	{
15312	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15313	rcStrict = VINF_SUCCESS;
15314	}
15315
15316	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
15317	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15318	return rcStrict;
15319	}
15320
15321
15322	/**
15323	* VM-exit helper for MOV from CRx (CRx read).
15324	*/
15325	static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
15326	{
15327	Assert(iCrReg < 16);
15328	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
15329
15330	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15331	AssertRCReturn(rc, rc);
15332
15333	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
15334	AssertMsg( rcStrict == VINF_SUCCESS
15335	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15336
15337	if (iGReg == X86_GREG_xSP)
15338	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
15339	else
15340	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15341	#ifdef VBOX_WITH_STATISTICS
15342	switch (iCrReg)
15343	{
15344	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
15345	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
15346	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
15347	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
15348	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
15349	}
15350	#endif
15351	Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
15352	return rcStrict;
15353	}
15354
15355
15356	/**
15357	* VM-exit helper for MOV to CRx (CRx write).
15358	*/
15359	static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
15360	{
15361	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15362	AssertRCReturn(rc, rc);
15363
15364	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
15365	AssertMsg( rcStrict == VINF_SUCCESS
15366	\|\| rcStrict == VINF_IEM_RAISED_XCPT
15367	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15368
15369	switch (iCrReg)
15370	{
15371	case 0:
15372	{
15373	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
15374	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
15375	Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
15376	break;
15377	}
15378
15379	case 2:
15380	{
15381	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
15382	/* Nothing to do here, CR2 it's not part of the VMCS. */
15383	break;
15384	}
15385
15386	case 3:
15387	{
15388	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
15389	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
15390	Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
15391	break;
15392	}
15393
15394	case 4:
15395	{
15396	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
15397	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
15398	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
15399	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
15400	break;
15401	}
15402
15403	case 8:
15404	{
15405	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
15406	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
15407	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
15408	break;
15409	}
15410
15411	default:
15412	AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
15413	break;
15414	}
15415
15416	if (rcStrict == VINF_IEM_RAISED_XCPT)
15417	{
15418	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15419	rcStrict = VINF_SUCCESS;
15420	}
15421	return rcStrict;
15422	}
15423
15424
15425	/**
15426	* VM-exit helper for handling host NMIs.
15427	*/
15428	static VBOXSTRICTRC hmR0VmxExitHostNmi(PVMCPU pVCpu)
15429	{
15430	VMXDispatchHostNmi();
15431
15432	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
15433	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
15434	return VINF_SUCCESS;
15435	}
15436
15437
15438	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15439	/** @name VMX instruction handlers.
15440	* @{
15441	*/
15442	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15443	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VMX instructions VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15444	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15445
15446	/**
15447	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
15448	*/
15449	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15450	{
15451	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15452
15453	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15454	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15455	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15456	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15457	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15458	AssertRCReturn(rc, rc);
15459
15460	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15461
15462	VMXVEXITINFO ExitInfo;
15463	RT_ZERO(ExitInfo);
15464	ExitInfo.uReason = pVmxTransient->uExitReason;
15465	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15466	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15467	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15468	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15469
15470	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
15471	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15472	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15473	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15474	{
15475	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15476	rcStrict = VINF_SUCCESS;
15477	}
15478	return rcStrict;
15479	}
15480
15481
15482	/**
15483	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
15484	*/
15485	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15486	{
15487	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15488
15489	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
15490	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15491	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15492	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15493	AssertRCReturn(rc, rc);
15494
15495	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15496
15497	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
15498	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15499	{
15500	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15501	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15502	}
15503	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15504	return rcStrict;
15505	}
15506
15507
15508	/**
15509	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
15510	*/
15511	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15512	{
15513	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15514
15515	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15516	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15517	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15518	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15519	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15520	AssertRCReturn(rc, rc);
15521
15522	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15523
15524	VMXVEXITINFO ExitInfo;
15525	RT_ZERO(ExitInfo);
15526	ExitInfo.uReason = pVmxTransient->uExitReason;
15527	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15528	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15529	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15530	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15531
15532	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
15533	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15534	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15535	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15536	{
15537	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15538	rcStrict = VINF_SUCCESS;
15539	}
15540	return rcStrict;
15541	}
15542
15543
15544	/**
15545	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
15546	*/
15547	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15548	{
15549	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15550
15551	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15552	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15553	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15554	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15555	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15556	AssertRCReturn(rc, rc);
15557
15558	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15559
15560	VMXVEXITINFO ExitInfo;
15561	RT_ZERO(ExitInfo);
15562	ExitInfo.uReason = pVmxTransient->uExitReason;
15563	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15564	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15565	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15566	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15567
15568	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
15569	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15570	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15571	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15572	{
15573	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15574	rcStrict = VINF_SUCCESS;
15575	}
15576	return rcStrict;
15577	}
15578
15579
15580	/**
15581	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
15582	*/
15583	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15584	{
15585	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15586
15587	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15588	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15589	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15590	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15591	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15592	AssertRCReturn(rc, rc);
15593
15594	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15595
15596	VMXVEXITINFO ExitInfo;
15597	RT_ZERO(ExitInfo);
15598	ExitInfo.uReason = pVmxTransient->uExitReason;
15599	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15600	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15601	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15602	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15603	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15604
15605	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
15606	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15607	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15608	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15609	{
15610	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15611	rcStrict = VINF_SUCCESS;
15612	}
15613	return rcStrict;
15614	}
15615
15616
15617	/**
15618	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
15619	*/
15620	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15621	{
15622	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15623
15624	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
15625	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15626	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15627	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15628	AssertRCReturn(rc, rc);
15629
15630	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15631
15632	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
15633	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15634	{
15635	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15636	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15637	}
15638	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15639	return rcStrict;
15640	}
15641
15642
15643	/**
15644	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
15645	*/
15646	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15647	{
15648	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15649
15650	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15651	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15652	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15653	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15654	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15655	AssertRCReturn(rc, rc);
15656
15657	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15658
15659	VMXVEXITINFO ExitInfo;
15660	RT_ZERO(ExitInfo);
15661	ExitInfo.uReason = pVmxTransient->uExitReason;
15662	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15663	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15664	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15665	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15666	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15667
15668	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
15669	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15670	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15671	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15672	{
15673	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15674	rcStrict = VINF_SUCCESS;
15675	}
15676	return rcStrict;
15677	}
15678
15679
15680	/**
15681	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
15682	*/
15683	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15684	{
15685	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15686
15687	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15688	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
15689	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15690	AssertRCReturn(rc, rc);
15691
15692	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15693
15694	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
15695	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15696	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
15697	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15698	{
15699	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15700	rcStrict = VINF_SUCCESS;
15701	}
15702	return rcStrict;
15703	}
15704
15705
15706	/**
15707	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
15708	*/
15709	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15710	{
15711	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15712
15713	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15714	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15715	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15716	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15717	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15718	AssertRCReturn(rc, rc);
15719
15720	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15721
15722	VMXVEXITINFO ExitInfo;
15723	RT_ZERO(ExitInfo);
15724	ExitInfo.uReason = pVmxTransient->uExitReason;
15725	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15726	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15727	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15728	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15729
15730	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
15731	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15732	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15733	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15734	{
15735	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15736	rcStrict = VINF_SUCCESS;
15737	}
15738	return rcStrict;
15739	}
15740
15741
15742	/**
15743	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
15744	*/
15745	HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15746	{
15747	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15748
15749	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15750	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15751	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15752	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15753	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15754	AssertRCReturn(rc, rc);
15755
15756	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15757
15758	VMXVEXITINFO ExitInfo;
15759	RT_ZERO(ExitInfo);
15760	ExitInfo.uReason = pVmxTransient->uExitReason;
15761	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15762	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15763	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15764	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15765
15766	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
15767	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15768	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15769	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15770	{
15771	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15772	rcStrict = VINF_SUCCESS;
15773	}
15774	return rcStrict;
15775	}
15776
15777	/** @} */
15778
15779	/** @name Nested-guest VM-exit handlers.
15780	* @{
15781	*/
15782	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15783	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15784	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15785
15786	/**
15787	* Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
15788	* Conditional VM-exit.
15789	*/
15790	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmiNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15791	{
15792	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15793
15794	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15795	AssertRCReturn(rc, rc);
15796
15797	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
15798	uint32_t const uExtIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
15799
15800	switch (uExtIntType)
15801	{
15802	/*
15803	* We shouldn't direct host physical NMIs to the nested-guest.
15804	*/
15805	case VMX_EXIT_INT_INFO_TYPE_NMI:
15806	return hmR0VmxExitHostNmi(pVCpu);
15807
15808	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
15809	{
15810	#if 0
15811	/* Page-faults are subject to masking using its error code. */
15812	uint32_t fXcptBitmap = pVmcs->u32XcptBitmap;
15813	if (uVector == X86_XCPT_PF)
15814	{
15815	uint32_t const fXcptPFMask = pVmcs->u32XcptPFMask;
15816	uint32_t const fXcptPFMatch = pVmcs->u32XcptPFMatch;
15817	if ((uErrCode & fXcptPFMask) != fXcptPFMatch)
15818	fXcptBitmap ^= RT_BIT(X86_XCPT_PF);
15819	}
15820
15821	/* Consult the exception bitmap for all other hardware exceptions. */
15822	Assert(uVector <= X86_XCPT_LAST);
15823	if (fXcptBitmap & RT_BIT(uVector))
15824	fIntercept = true;
15825	#endif
15826	break;
15827	}
15828
15829	/*
15830	* This should only happen when "acknowledge external interrupts on VM-exit" is set.
15831	* We don't set it when executing guests or nested-guests.
15832	*/
15833	case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
15834	RT_FALL_THRU();
15835	default:
15836	{
15837	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
15838	return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
15839	}
15840	}
15841
15842	return VERR_NOT_IMPLEMENTED;
15843	}
15844
15845
15846	/**
15847	* Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
15848	* Unconditional VM-exit.
15849	*/
15850	HMVMX_EXIT_DECL hmR0VmxExitTripleFaultNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15851	{
15852	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15853	return IEMExecVmxVmexitTripleFault(pVCpu);
15854	}
15855
15856
15857	/**
15858	* Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
15859	*/
15860	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15861	{
15862	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15863
15864	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
15865	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
15866	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
15867	}
15868
15869
15870	/**
15871	* Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
15872	*/
15873	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15874	{
15875	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15876
15877	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
15878	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
15879	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
15880	}
15881
15882
15883	/**
15884	* Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
15885	* Unconditional VM-exit.
15886	*/
15887	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitchNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15888	{
15889	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15890
15891	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15892	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15893	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15894	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15895	AssertRCReturn(rc, rc);
15896
15897	VMXVEXITINFO ExitInfo;
15898	RT_ZERO(ExitInfo);
15899	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15900	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15901
15902	VMXVEXITEVENTINFO ExitEventInfo;
15903	RT_ZERO(ExitInfo);
15904	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
15905	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
15906	return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
15907	}
15908
15909
15910	/**
15911	* Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
15912	*/
15913	HMVMX_EXIT_DECL hmR0VmxExitHltNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15914	{
15915	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15916
15917	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
15918	{
15919	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15920	AssertRCReturn(rc, rc);
15921	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
15922	}
15923	return hmR0VmxExitHlt(pVCpu, pVmxTransient);
15924	}
15925
15926
15927	/**
15928	* Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
15929	*/
15930	HMVMX_EXIT_DECL hmR0VmxExitInvlpgNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15931	{
15932	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15933
15934	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
15935	{
15936	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15937	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15938	AssertRCReturn(rc, rc);
15939
15940	VMXVEXITINFO ExitInfo;
15941	RT_ZERO(ExitInfo);
15942	ExitInfo.uReason = pVmxTransient->uExitReason;
15943	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15944	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15945	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
15946	}
15947	return hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
15948	}
15949
15950
15951	/**
15952	* Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
15953	*/
15954	HMVMX_EXIT_DECL hmR0VmxExitRdpmcNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15955	{
15956	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15957
15958	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
15959	{
15960	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15961	AssertRCReturn(rc, rc);
15962	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
15963	}
15964	return hmR0VmxExitRdpmc(pVCpu, pVmxTransient);
15965	}
15966
15967
15968	/**
15969	* Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
15970	* (VMX_EXIT_VMWRITE). Conditional VM-exit.
15971	*/
15972	HMVMX_EXIT_DECL hmR0VmxExitVmreadVmwriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15973	{
15974	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15975
15976	Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD
15977	\|\| pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
15978
15979	int rc = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15980	AssertRCReturn(rc, rc);
15981
15982	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
15983	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
15984	uint64_t u64FieldEnc = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
15985
15986	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
15987	if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
15988	u64FieldEnc &= UINT64_C(0xffffffff);
15989
15990	if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64FieldEnc))
15991	{
15992	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15993	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15994	AssertRCReturn(rc, rc);
15995
15996	VMXVEXITINFO ExitInfo;
15997	RT_ZERO(ExitInfo);
15998	ExitInfo.uReason = pVmxTransient->uExitReason;
15999	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16000	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16001	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16002	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16003	}
16004
16005	if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
16006	return hmR0VmxExitVmread(pVCpu, pVmxTransient);
16007	return hmR0VmxExitVmwrite(pVCpu, pVmxTransient);
16008	}
16009
16010
16011	/**
16012	* Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
16013	*/
16014	HMVMX_EXIT_DECL hmR0VmxExitRdtscNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16015	{
16016	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16017
16018	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
16019	{
16020	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16021	AssertRCReturn(rc, rc);
16022	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16023	}
16024
16025	return hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
16026	}
16027
16028
16029	/**
16030	* Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
16031	* Conditional VM-exit.
16032	*/
16033	HMVMX_EXIT_DECL hmR0VmxExitMovCRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16034	{
16035	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16036
16037	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16038	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16039	AssertRCReturn(rc, rc);
16040
16041	VBOXSTRICTRC rcStrict;
16042	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
16043	switch (uAccessType)
16044	{
16045	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
16046	{
16047	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16048	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16049	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16050	uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
16051	if (CPUMIsGuestVmxMovToCr0Cr4InterceptSet(pVCpu, &pVCpu->cpum.GstCtx, iCrReg, uNewCrX))
16052	{
16053	VMXVEXITINFO ExitInfo;
16054	RT_ZERO(ExitInfo);
16055	ExitInfo.uReason = pVmxTransient->uExitReason;
16056	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16057	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16058	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16059	}
16060	else
16061	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
16062	break;
16063	}
16064
16065	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
16066	{
16067	/*
16068	* CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
16069	* CR2 reads do not cause a VM-exit.
16070	* CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
16071	* CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
16072	*/
16073	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16074	if ( iCrReg == 3
16075	\|\| iCrReg == 8)
16076	{
16077	static const uint32_t s_aCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
16078	0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
16079	uint32_t const uIntercept = s_aCrXReadIntercepts[iCrReg];
16080	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uIntercept))
16081	{
16082	VMXVEXITINFO ExitInfo;
16083	RT_ZERO(ExitInfo);
16084	ExitInfo.uReason = pVmxTransient->uExitReason;
16085	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16086	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16087	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16088	}
16089	else
16090	{
16091	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16092	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
16093	}
16094	}
16095	else
16096	{
16097	AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
16098	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
16099	}
16100	break;
16101	}
16102
16103	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
16104	{
16105	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
16106	Assert(pVmcsNstGst);
16107	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
16108	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
16109	if ( (uGstHostMask & X86_CR0_TS)
16110	&& (uReadShadow & X86_CR0_TS))
16111	{
16112	VMXVEXITINFO ExitInfo;
16113	RT_ZERO(ExitInfo);
16114	ExitInfo.uReason = pVmxTransient->uExitReason;
16115	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16116	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16117	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16118	}
16119	else
16120	rcStrict = hmR0VmxExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr);
16121	break;
16122	}
16123
16124	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
16125	{
16126	RTGCPTR GCPtrEffDst;
16127	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
16128	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
16129	if (fMemOperand)
16130	{
16131	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
16132	AssertRCReturn(rc, rc);
16133	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
16134	}
16135	else
16136	GCPtrEffDst = NIL_RTGCPTR;
16137
16138	if (CPUMIsGuestVmxLmswInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uNewMsw))
16139	{
16140	VMXVEXITINFO ExitInfo;
16141	RT_ZERO(ExitInfo);
16142	ExitInfo.uReason = pVmxTransient->uExitReason;
16143	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16144	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
16145	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16146	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16147	}
16148	else
16149	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, uNewMsw, GCPtrEffDst);
16150	break;
16151	}
16152
16153	default:
16154	{
16155	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
16156	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
16157	}
16158	}
16159
16160	if (rcStrict == VINF_IEM_RAISED_XCPT)
16161	{
16162	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16163	rcStrict = VINF_SUCCESS;
16164	}
16165	return rcStrict;
16166	}
16167
16168
16169	/**
16170	* Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
16171	* Conditional VM-exit.
16172	*/
16173	HMVMX_EXIT_DECL hmR0VmxExitMovDRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16174	{
16175	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16176
16177	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
16178	{
16179	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16180	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16181	AssertRCReturn(rc, rc);
16182
16183	VMXVEXITINFO ExitInfo;
16184	RT_ZERO(ExitInfo);
16185	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16186	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16187	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16188	}
16189	return hmR0VmxExitMovDRx(pVCpu, pVmxTransient);
16190	}
16191
16192
16193	/**
16194	* Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
16195	* Conditional VM-exit.
16196	*/
16197	HMVMX_EXIT_DECL hmR0VmxExitIoInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16198	{
16199	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16200
16201	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16202	AssertRCReturn(rc, rc);
16203
16204	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
16205	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
16206	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
16207
16208	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
16209	uint8_t const cbAccess = s_aIOSizes[uIOSize];
16210	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
16211	{
16212	/*
16213	* IN/OUT instruction:
16214	* - Provides VM-exit instruction length.
16215	*
16216	* INS/OUTS instruction:
16217	* - Provides VM-exit instruction length.
16218	* - Provides Guest-linear address.
16219	* - Optionally provides VM-exit instruction info (depends on CPU feature).
16220	*/
16221	PVM pVM = pVCpu->CTX_SUFF(pVM);
16222	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16223	AssertRCReturn(rc, rc);
16224
16225	/* Make sure we don't use stale/uninitialized VMX-transient info. below. */
16226	pVmxTransient->ExitInstrInfo.u = 0;
16227	pVmxTransient->uGuestLinearAddr = 0;
16228
16229	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
16230	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
16231	if (fIOString)
16232	{
16233	rc \|= hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
16234	if (fVmxInsOutsInfo)
16235	{
16236	Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
16237	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16238	}
16239	}
16240	AssertRCReturn(rc, rc);
16241
16242	VMXVEXITINFO ExitInfo;
16243	RT_ZERO(ExitInfo);
16244	ExitInfo.uReason = pVmxTransient->uExitReason;
16245	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16246	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16247	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16248	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
16249	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16250	}
16251	return hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
16252	}
16253
16254
16255	/**
16256	* Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
16257	*/
16258	HMVMX_EXIT_DECL hmR0VmxExitRdmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16259	{
16260	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16261
16262	uint32_t fMsrpm;
16263	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
16264	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
16265	else
16266	fMsrpm = VMXMSRPM_EXIT_RD;
16267
16268	if (fMsrpm & VMXMSRPM_EXIT_RD)
16269	{
16270	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16271	AssertRCReturn(rc, rc);
16272	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16273	}
16274	return hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
16275	}
16276
16277
16278	/**
16279	* Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
16280	*/
16281	HMVMX_EXIT_DECL hmR0VmxExitWrmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16282	{
16283	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16284
16285	uint32_t fMsrpm;
16286	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
16287	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
16288	else
16289	fMsrpm = VMXMSRPM_EXIT_WR;
16290
16291	if (fMsrpm & VMXMSRPM_EXIT_WR)
16292	{
16293	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16294	AssertRCReturn(rc, rc);
16295	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16296	}
16297	return hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
16298	}
16299
16300
16301	/**
16302	* Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
16303	*/
16304	HMVMX_EXIT_DECL hmR0VmxExitMwaitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16305	{
16306	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16307
16308	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
16309	{
16310	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16311	AssertRCReturn(rc, rc);
16312	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16313	}
16314	return hmR0VmxExitMwait(pVCpu, pVmxTransient);
16315	}
16316
16317
16318	/**
16319	* Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
16320	* VM-exit.
16321	*/
16322	HMVMX_EXIT_DECL hmR0VmxExitMtfNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16323	{
16324	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16325
16326	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
16327	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16328	}
16329
16330
16331	/**
16332	* Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
16333	*/
16334	HMVMX_EXIT_DECL hmR0VmxExitMonitorNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16335	{
16336	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16337
16338	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
16339	{
16340	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16341	AssertRCReturn(rc, rc);
16342	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16343	}
16344	return hmR0VmxExitMonitor(pVCpu, pVmxTransient);
16345	}
16346
16347
16348	/**
16349	* Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
16350	*/
16351	HMVMX_EXIT_DECL hmR0VmxExitPauseNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16352	{
16353	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16354
16355	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
16356	* PAUSE when executing a nested-guest? If it does not, we would not need
16357	* to check for the intercepts here. Just call VM-exit... */
16358
16359	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
16360	if ( CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
16361	\|\| CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
16362	{
16363	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16364	AssertRCReturn(rc, rc);
16365	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16366	}
16367	return hmR0VmxExitPause(pVCpu, pVmxTransient);
16368	}
16369
16370
16371	/**
16372	* Nested-guest VM-exit handler for when the TPR value is lowered below the
16373	* specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
16374	*/
16375	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThresholdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16376	{
16377	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16378
16379	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
16380	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16381	return hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient);
16382	}
16383
16384
16385	/**
16386	* Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
16387	* VM-exit.
16388	*/
16389	HMVMX_EXIT_DECL hmR0VmxExitApicAccessNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16390	{
16391	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16392
16393	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
16394	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16395	AssertRCReturn(rc, rc);
16396
16397	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
16398	}
16399
16400
16401	/**
16402	* Nested-guest VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
16403	* (VMX_EXIT_GDTR_IDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
16404	* Conditional VM-exit.
16405	*/
16406	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccessNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16407	{
16408	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16409
16410	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
16411	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16412	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16413	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16414	AssertRCReturn(rc, rc);
16415
16416	VMXVEXITINFO ExitInfo;
16417	RT_ZERO(ExitInfo);
16418	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16419	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16420	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16421	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16422	}
16423
16424
16425	/**
16426	* Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
16427	*/
16428	HMVMX_EXIT_DECL hmR0VmxExitRdtscpNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16429	{
16430	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16431
16432	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
16433	{
16434	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
16435	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16436	AssertRCReturn(rc, rc);
16437	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16438	}
16439	return hmR0VmxExitRdtscp(pVCpu, pVmxTransient);
16440	}
16441
16442
16443	/**
16444	* Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
16445	*/
16446	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16447	{
16448	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16449
16450	if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
16451	{
16452	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16453	AssertRCReturn(rc, rc);
16454	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16455	}
16456	return hmR0VmxExitWbinvd(pVCpu, pVmxTransient);
16457	}
16458
16459
16460	/**
16461	* Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
16462	*/
16463	HMVMX_EXIT_DECL hmR0VmxExitInvpcidNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16464	{
16465	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16466
16467	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
16468	{
16469	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
16470	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16471	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16472	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16473	AssertRCReturn(rc, rc);
16474
16475	VMXVEXITINFO ExitInfo;
16476	RT_ZERO(ExitInfo);
16477	ExitInfo.uReason = pVmxTransient->uExitReason;
16478	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16479	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16480	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16481	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16482	}
16483	return hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
16484	}
16485
16486
16487	/**
16488	* Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
16489	* and only provide the instruction length.
16490	*
16491	* Unconditional VM-exit.
16492	*/
16493	HMVMX_EXIT_DECL hmR0VmxExitInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16494	{
16495	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16496
16497	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16498	AssertRCReturn(rc, rc);
16499	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16500	}
16501
16502
16503	/**
16504	* Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
16505	* but provide instruction length as well as more information.
16506	*
16507	* Unconditional VM-exit.
16508	*/
16509	HMVMX_EXIT_DECL hmR0VmxExitInstrWithInfoNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16510	{
16511	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16512
16513	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16514	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16515	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16516	AssertRCReturn(rc, rc);
16517
16518	VMXVEXITINFO ExitInfo;
16519	RT_ZERO(ExitInfo);
16520	ExitInfo.uReason = pVmxTransient->uExitReason;
16521	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16522	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16523	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16524	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16525	}
16526
16527	/** @} */
16528
16529	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
16530

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 78928

Download in other formats: