HMVMXR0.cpp@ 78927

Last change on this file since 78927 was 78887, checked in by vboxsync, 6 years ago
VMM/HMVMXR0: Nested VMX: bugref:9180 TPR-below threshold VM-exit handling. Assertion.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 685.2 KB

Line
1	/* $Id: HMVMXR0.cpp 78887 2019-05-31 05:09:51Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27
28	#include <VBox/vmm/pdmapi.h>
29	#include <VBox/vmm/dbgf.h>
30	#include <VBox/vmm/iem.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#include <VBox/vmm/em.h>
35	#include <VBox/vmm/gim.h>
36	#include <VBox/vmm/apic.h>
37	#ifdef VBOX_WITH_REM
38	# include <VBox/vmm/rem.h>
39	#endif
40	#include "HMInternal.h"
41	#include <VBox/vmm/vm.h>
42	#include <VBox/vmm/hmvmxinline.h>
43	#include "HMVMXR0.h"
44	#include "dtrace/VBoxVMM.h"
45
46	#ifdef DEBUG_ramshankar
47	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
48	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
49	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
50	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
51	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
52	# define HMVMX_ALWAYS_TRAP_PF
53	# define HMVMX_ALWAYS_FLUSH_TLB
54	# define HMVMX_ALWAYS_SWAP_EFER
55	#endif
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	/** Use the function table. */
62	#define HMVMX_USE_FUNCTION_TABLE
63
64	/** Determine which tagged-TLB flush handler to use. */
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70	/** @name HMVMX_READ_XXX
71	* Flags to skip redundant reads of some common VMCS fields that are not part of
72	* the guest-CPU or VCPU state but are needed while handling VM-exits.
73	*/
74	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
75	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
76	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
77	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
78	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
79	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
80	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
81	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
82	/** @} */
83
84	/**
85	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
86	* guest using hardware-assisted VMX.
87	*
88	* This excludes state like GPRs (other than RSP) which are always are
89	* swapped and restored across the world-switch and also registers like EFER,
90	* MSR which cannot be modified by the guest without causing a VM-exit.
91	*/
92	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
93	\| CPUMCTX_EXTRN_RFLAGS \
94	\| CPUMCTX_EXTRN_RSP \
95	\| CPUMCTX_EXTRN_SREG_MASK \
96	\| CPUMCTX_EXTRN_TABLE_MASK \
97	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
98	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
99	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
100	\| CPUMCTX_EXTRN_TSC_AUX \
101	\| CPUMCTX_EXTRN_OTHER_MSRS \
102	\| CPUMCTX_EXTRN_CR0 \
103	\| CPUMCTX_EXTRN_CR3 \
104	\| CPUMCTX_EXTRN_CR4 \
105	\| CPUMCTX_EXTRN_DR7 \
106	\| CPUMCTX_EXTRN_HM_VMX_MASK)
107
108	/**
109	* Exception bitmap mask for real-mode guests (real-on-v86).
110	*
111	* We need to intercept all exceptions manually except:
112	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
113	* due to bugs in Intel CPUs.
114	* - \#PF need not be intercepted even in real-mode if we have nested paging
115	* support.
116	*/
117	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
118	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
119	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
120	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
121	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
122	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
123	\| RT_BIT(X86_XCPT_XF))
124
125	/** Maximum VM-instruction error number. */
126	#define HMVMX_INSTR_ERROR_MAX 28
127
128	/** Profiling macro. */
129	#ifdef HM_PROFILE_EXIT_DISPATCH
130	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
131	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
132	#else
133	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
134	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
135	#endif
136
137	/** Assert that preemption is disabled or covered by thread-context hooks. */
138	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
139	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
140
141	/** Assert that we haven't migrated CPUs when thread-context hooks are not
142	* used. */
143	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
144	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
145	("Illegal migration! Entered on CPU %u Current %u\n", \
146	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
147
148	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
149	* context. */
150	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
151	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
152	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
153
154	/** Helper macro for VM-exit handlers called unexpectedly. */
155	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
156	do { \
157	(a_pVCpu)->hm.s.u32HMError = (a_HmError); \
158	return VERR_VMX_UNEXPECTED_EXIT; \
159	} while (0)
160
161	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
162	/** Macro that does the necessary privilege checks and intercepted VM-exits for
163	* guests that attempted to execute a VMX instruction. */
164	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
165	do \
166	{ \
167	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
168	if (rcStrictTmp == VINF_SUCCESS) \
169	{ /* likely */ } \
170	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
171	{ \
172	Assert((a_pVCpu)->hm.s.Event.fPending); \
173	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
174	return VINF_SUCCESS; \
175	} \
176	else \
177	{ \
178	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
179	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
180	} \
181	} while (0)
182
183	/** Macro that decodes a memory operand for an instruction VM-exit. */
184	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
185	do \
186	{ \
187	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
188	(a_pGCPtrEffAddr)); \
189	if (rcStrictTmp == VINF_SUCCESS) \
190	{ /* likely */ } \
191	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
192	{ \
193	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
194	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
195	NOREF(uXcptTmp); \
196	return VINF_SUCCESS; \
197	} \
198	else \
199	{ \
200	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
201	return rcStrictTmp; \
202	} \
203	} while (0)
204
205	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
206
207
208	/*********************************************************************************************************************************
209	* Structures and Typedefs *
210	*********************************************************************************************************************************/
211	/**
212	* VMX transient state.
213	*
214	* A state structure for holding miscellaneous information across
215	* VMX non-root operation and restored after the transition.
216	*/
217	typedef struct VMXTRANSIENT
218	{
219	/** The host's rflags/eflags. */
220	RTCCUINTREG fEFlags;
221	#if HC_ARCH_BITS == 32
222	uint32_t u32Alignment0;
223	#endif
224	/** The guest's TPR value used for TPR shadowing. */
225	uint8_t u8GuestTpr;
226	/** Alignment. */
227	uint8_t abAlignment0[7];
228
229	/** The basic VM-exit reason. */
230	uint16_t uExitReason;
231	/** Alignment. */
232	uint16_t u16Alignment0;
233	/** The VM-exit interruption error code. */
234	uint32_t uExitIntErrorCode;
235	/** The VM-exit exit code qualification. */
236	uint64_t uExitQual;
237	/** The Guest-linear address. */
238	uint64_t uGuestLinearAddr;
239
240	/** The VM-exit interruption-information field. */
241	uint32_t uExitIntInfo;
242	/** The VM-exit instruction-length field. */
243	uint32_t cbInstr;
244	/** The VM-exit instruction-information field. */
245	VMXEXITINSTRINFO ExitInstrInfo;
246	/** Whether the VM-entry failed or not. */
247	bool fVMEntryFailed;
248	/** Whether we are currently executing a nested-guest. */
249	bool fIsNestedGuest;
250	/** Alignment. */
251	uint8_t abAlignment1[2];
252
253	/** The VM-entry interruption-information field. */
254	uint32_t uEntryIntInfo;
255	/** The VM-entry exception error code field. */
256	uint32_t uEntryXcptErrorCode;
257	/** The VM-entry instruction length field. */
258	uint32_t cbEntryInstr;
259
260	/** IDT-vectoring information field. */
261	uint32_t uIdtVectoringInfo;
262	/** IDT-vectoring error code. */
263	uint32_t uIdtVectoringErrorCode;
264
265	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
266	uint32_t fVmcsFieldsRead;
267
268	/** Whether the guest debug state was active at the time of VM-exit. */
269	bool fWasGuestDebugStateActive;
270	/** Whether the hyper debug state was active at the time of VM-exit. */
271	bool fWasHyperDebugStateActive;
272	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
273	bool fUpdatedTscOffsettingAndPreemptTimer;
274	/** Whether the VM-exit was caused by a page-fault during delivery of a
275	* contributory exception or a page-fault. */
276	bool fVectoringDoublePF;
277	/** Whether the VM-exit was caused by a page-fault during delivery of an
278	* external interrupt or NMI. */
279	bool fVectoringPF;
280	bool afAlignment0[3];
281
282	/** The VMCS info. object. */
283	PVMXVMCSINFO pVmcsInfo;
284	} VMXTRANSIENT;
285	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
286	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
287	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
288	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
289	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
290	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
291	/** Pointer to VMX transient state. */
292	typedef VMXTRANSIENT *PVMXTRANSIENT;
293
294	/**
295	* Memory operand read or write access.
296	*/
297	typedef enum VMXMEMACCESS
298	{
299	VMXMEMACCESS_READ = 0,
300	VMXMEMACCESS_WRITE = 1
301	} VMXMEMACCESS;
302
303	/**
304	* VMX VM-exit handler.
305	*
306	* @returns Strict VBox status code (i.e. informational status codes too).
307	* @param pVCpu The cross context virtual CPU structure.
308	* @param pVmxTransient The VMX-transient structure.
309	*/
310	#ifndef HMVMX_USE_FUNCTION_TABLE
311	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
312	#else
313	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
314	/** Pointer to VM-exit handler. */
315	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
316	#endif
317
318	/**
319	* VMX VM-exit handler, non-strict status code.
320	*
321	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
322	*
323	* @returns VBox status code, no informational status code returned.
324	* @param pVCpu The cross context virtual CPU structure.
325	* @param pVmxTransient The VMX-transient structure.
326	*
327	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
328	* use of that status code will be replaced with VINF_EM_SOMETHING
329	* later when switching over to IEM.
330	*/
331	#ifndef HMVMX_USE_FUNCTION_TABLE
332	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
333	#else
334	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
335	#endif
336
337
338	/*********************************************************************************************************************************
339	* Internal Functions *
340	*********************************************************************************************************************************/
341	#ifndef HMVMX_USE_FUNCTION_TABLE
342	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
343	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
344	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
345	#else
346	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
347	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
348	#endif
349	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
350	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
351	#endif
352
353	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
354	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
355	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
356	#endif
357
358	/** @name VM-exit handlers.
359	* @{
360	*/
361	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
362	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
363	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
364	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
365	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
366	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
367	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
368	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
369	static FNVMXEXITHANDLER hmR0VmxExitHlt;
370	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
371	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
372	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
373	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
374	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
375	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
376	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
377	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
378	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
379	static FNVMXEXITHANDLER hmR0VmxExitVmread;
380	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
381	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
382	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
383	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
384	static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
385	#endif
386	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
387	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
388	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
389	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
390	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
391	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
392	static FNVMXEXITHANDLER hmR0VmxExitMwait;
393	static FNVMXEXITHANDLER hmR0VmxExitMtf;
394	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
395	static FNVMXEXITHANDLER hmR0VmxExitPause;
396	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
397	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
398	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
399	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
400	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
401	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
402	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
403	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
404	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
405	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
406	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
407	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
408	/** @} */
409
410	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
411	/** @name Nested-guest VM-exit handlers.
412	* @{
413	*/
414	//static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
415	//static FNVMXEXITHANDLER hmR0VmxExitExtIntNested;
416	static FNVMXEXITHANDLER hmR0VmxExitTripleFaultNested;
417	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindowNested;
418	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindowNested;
419	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitchNested;
420	//static FNVMXEXITHANDLER hmR0VmxExitCpuid;
421	//static FNVMXEXITHANDLER hmR0VmxExitGetsec;
422	static FNVMXEXITHANDLER hmR0VmxExitHltNested;
423	//static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
424	static FNVMXEXITHANDLER hmR0VmxExitInvlpgNested;
425	static FNVMXEXITHANDLER hmR0VmxExitRdpmcNested;
426	//static FNVMXEXITHANDLER hmR0VmxExitVmcall;
427	//static FNVMXEXITHANDLER hmR0VmxExitVmclear;
428	//static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
429	//static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
430	//static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
431	static FNVMXEXITHANDLER hmR0VmxExitVmreadVmwriteNested;
432	//static FNVMXEXITHANDLER hmR0VmxExitVmresume;
433	//static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
434	//static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
435	//static FNVMXEXITHANDLER hmR0VmxExitVmxon;
436	//static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
437	static FNVMXEXITHANDLER hmR0VmxExitRdtscNested;
438	static FNVMXEXITHANDLER hmR0VmxExitMovCRxNested;
439	static FNVMXEXITHANDLER hmR0VmxExitMovDRxNested;
440	static FNVMXEXITHANDLER hmR0VmxExitIoInstrNested;
441	static FNVMXEXITHANDLER hmR0VmxExitRdmsrNested;
442	static FNVMXEXITHANDLER hmR0VmxExitWrmsrNested;
443	static FNVMXEXITHANDLER hmR0VmxExitMwaitNested;
444	static FNVMXEXITHANDLER hmR0VmxExitMtfNested;
445	static FNVMXEXITHANDLER hmR0VmxExitMonitorNested;
446	static FNVMXEXITHANDLER hmR0VmxExitPauseNested;
447	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThresholdNested;
448	static FNVMXEXITHANDLER hmR0VmxExitApicAccessNested;
449	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccessNested;
450	//static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
451	//static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
452	static FNVMXEXITHANDLER hmR0VmxExitRdtscpNested;
453	//static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
454	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvdNested;
455	//static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
456	//static FNVMXEXITHANDLER hmR0VmxExitErrUnexpected;
457	static FNVMXEXITHANDLER hmR0VmxExitInvpcidNested;
458	//static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
459	//static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
460	//static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
461	static FNVMXEXITHANDLER hmR0VmxExitInstrNested;
462	static FNVMXEXITHANDLER hmR0VmxExitInstrWithInfoNested;
463	/** @} */
464	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
465
466	/** @name Helpers for hardware exceptions VM-exit handlers.
467	* @{
468	*/
469	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
470	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
471	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
472	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
473	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
474	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
475	static VBOXSTRICTRC hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
476	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst);
477	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr);
478	static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg);
479	static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg);
480	static VBOXSTRICTRC hmR0VmxExitHostNmi(PVMCPU pVCpu);
481	/** @} */
482
483
484	/*********************************************************************************************************************************
485	* Global Variables *
486	*********************************************************************************************************************************/
487	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
488	static const uint32_t g_aVmcsCacheSegBase[] =
489	{
490	VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
491	VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
492	VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
493	VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
494	VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
495	VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
496	};
497	AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
498	#endif
499	static const uint32_t g_aVmcsSegBase[] =
500	{
501	VMX_VMCS_GUEST_ES_BASE,
502	VMX_VMCS_GUEST_CS_BASE,
503	VMX_VMCS_GUEST_SS_BASE,
504	VMX_VMCS_GUEST_DS_BASE,
505	VMX_VMCS_GUEST_FS_BASE,
506	VMX_VMCS_GUEST_GS_BASE
507	};
508	static const uint32_t g_aVmcsSegSel[] =
509	{
510	VMX_VMCS16_GUEST_ES_SEL,
511	VMX_VMCS16_GUEST_CS_SEL,
512	VMX_VMCS16_GUEST_SS_SEL,
513	VMX_VMCS16_GUEST_DS_SEL,
514	VMX_VMCS16_GUEST_FS_SEL,
515	VMX_VMCS16_GUEST_GS_SEL
516	};
517	static const uint32_t g_aVmcsSegLimit[] =
518	{
519	VMX_VMCS32_GUEST_ES_LIMIT,
520	VMX_VMCS32_GUEST_CS_LIMIT,
521	VMX_VMCS32_GUEST_SS_LIMIT,
522	VMX_VMCS32_GUEST_DS_LIMIT,
523	VMX_VMCS32_GUEST_FS_LIMIT,
524	VMX_VMCS32_GUEST_GS_LIMIT
525	};
526	static const uint32_t g_aVmcsSegAttr[] =
527	{
528	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
529	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
530	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
531	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
532	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
533	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
534	};
535	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
536	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
537	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
538	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
539
540	#ifdef HMVMX_USE_FUNCTION_TABLE
541	/**
542	* VMX_EXIT dispatch table.
543	*/
544	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
545	{
546	/* 0 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
547	/* 1 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
548	/* 2 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
549	/* 3 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitErrUnexpected,
550	/* 4 VMX_EXIT_SIPI */ hmR0VmxExitErrUnexpected,
551	/* 5 VMX_EXIT_IO_SMI */ hmR0VmxExitErrUnexpected,
552	/* 6 VMX_EXIT_SMI */ hmR0VmxExitErrUnexpected,
553	/* 7 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
554	/* 8 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
555	/* 9 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
556	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
557	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
558	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
559	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
560	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
561	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
562	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
563	/* 17 VMX_EXIT_RSM */ hmR0VmxExitErrUnexpected,
564	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
565	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
566	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
567	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
568	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
569	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
570	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
571	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
572	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
573	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
574	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
575	#else
576	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
577	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
578	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
579	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
580	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
581	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
582	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
583	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
584	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
585	#endif
586	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
587	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
588	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
589	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
590	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
591	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
592	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrUnexpected,
593	/* 35 UNDEFINED */ hmR0VmxExitErrUnexpected,
594	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
595	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
596	/* 38 UNDEFINED */ hmR0VmxExitErrUnexpected,
597	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
598	/* 40 VMX_EXIT_PAUSE */ hmR0VmxExitPause,
599	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUnexpected,
600	/* 42 UNDEFINED */ hmR0VmxExitErrUnexpected,
601	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
602	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
603	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ hmR0VmxExitErrUnexpected,
604	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitErrUnexpected,
605	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitErrUnexpected,
606	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
607	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
608	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
609	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
610	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
611	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
612	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInvvpid,
613	#else
614	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
615	#endif
616	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
617	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
618	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUnexpected,
619	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitErrUnexpected,
620	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
621	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitErrUnexpected,
622	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUnexpected,
623	/* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUnexpected,
624	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUnexpected,
625	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitErrUnexpected,
626	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitErrUnexpected,
627	/* 65 UNDEFINED */ hmR0VmxExitErrUnexpected,
628	/* 66 VMX_EXIT_SPP_EVENT */ hmR0VmxExitErrUnexpected,
629	/* 67 VMX_EXIT_UMWAIT */ hmR0VmxExitErrUnexpected,
630	/* 68 VMX_EXIT_TPAUSE */ hmR0VmxExitErrUnexpected,
631	};
632	#endif /* HMVMX_USE_FUNCTION_TABLE */
633
634	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
635	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
636	{
637	/* 0 */ "(Not Used)",
638	/* 1 */ "VMCALL executed in VMX root operation.",
639	/* 2 */ "VMCLEAR with invalid physical address.",
640	/* 3 */ "VMCLEAR with VMXON pointer.",
641	/* 4 */ "VMLAUNCH with non-clear VMCS.",
642	/* 5 */ "VMRESUME with non-launched VMCS.",
643	/* 6 */ "VMRESUME after VMXOFF",
644	/* 7 */ "VM-entry with invalid control fields.",
645	/* 8 */ "VM-entry with invalid host state fields.",
646	/* 9 */ "VMPTRLD with invalid physical address.",
647	/* 10 */ "VMPTRLD with VMXON pointer.",
648	/* 11 */ "VMPTRLD with incorrect revision identifier.",
649	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
650	/* 13 */ "VMWRITE to read-only VMCS component.",
651	/* 14 */ "(Not Used)",
652	/* 15 */ "VMXON executed in VMX root operation.",
653	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
654	/* 17 */ "VM-entry with non-launched executing VMCS.",
655	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
656	/* 19 */ "VMCALL with non-clear VMCS.",
657	/* 20 */ "VMCALL with invalid VM-exit control fields.",
658	/* 21 */ "(Not Used)",
659	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
660	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
661	/* 24 */ "VMCALL with invalid SMM-monitor features.",
662	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
663	/* 26 */ "VM-entry with events blocked by MOV SS.",
664	/* 27 */ "(Not Used)",
665	/* 28 */ "Invalid operand to INVEPT/INVVPID."
666	};
667	#endif /* VBOX_STRICT */
668
669
670	/**
671	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
672	*
673	* Any bit set in this mask is owned by the host/hypervisor and would cause a
674	* VM-exit when modified by the guest.
675	*
676	* @returns The static CR0 guest/host mask.
677	* @param pVCpu The cross context virtual CPU structure.
678	*/
679	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PCVMCPU pVCpu)
680	{
681	/*
682	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
683	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
684	*/
685	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
686	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
687	* and @bugref{6944}. */
688	PVM pVM = pVCpu->CTX_SUFF(pVM);
689	return ( X86_CR0_PE
690	\| X86_CR0_NE
691	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
692	\| X86_CR0_PG
693	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
694	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
695	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
696	}
697
698
699	/**
700	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
701	*
702	* Any bit set in this mask is owned by the host/hypervisor and would cause a
703	* VM-exit when modified by the guest.
704	*
705	* @returns The static CR4 guest/host mask.
706	* @param pVCpu The cross context virtual CPU structure.
707	*/
708	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PCVMCPU pVCpu)
709	{
710	/*
711	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
712	* these bits are reserved on hardware that does not support them. Since the
713	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
714	* these bits and handle it depending on whether we expose them to the guest.
715	*/
716	PVM pVM = pVCpu->CTX_SUFF(pVM);
717	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
718	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
719	return ( X86_CR4_VMXE
720	\| X86_CR4_VME
721	\| X86_CR4_PAE
722	\| X86_CR4_PGE
723	\| X86_CR4_PSE
724	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
725	\| (fPcid ? X86_CR4_PCIDE : 0));
726	}
727
728
729	/**
730	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
731	* area.
732	*
733	* @returns @c true if it's different, @c false otherwise.
734	* @param pVmcsInfo The VMCS info. object.
735	*/
736	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
737	{
738	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
739	&& pVmcsInfo->pvGuestMsrStore);
740	}
741
742
743	/**
744	* Adds one or more exceptions to the exception bitmap and commits it to the current
745	* VMCS.
746	*
747	* @returns VBox status code.
748	* @param pVmxTransient The VMX-transient structure.
749	* @param uXcptMask The exception(s) to add.
750	*/
751	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
752	{
753	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
754	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
755	if ((uXcptBitmap & uXcptMask) != uXcptMask)
756	{
757	uXcptBitmap \|= uXcptMask;
758	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
759	AssertRCReturn(rc, rc);
760	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
761	}
762	return VINF_SUCCESS;
763	}
764
765
766	/**
767	* Adds an exception to the exception bitmap and commits it to the current VMCS.
768	*
769	* @returns VBox status code.
770	* @param pVmxTransient The VMX-transient structure.
771	* @param uXcpt The exception to add.
772	*/
773	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
774	{
775	Assert(uXcpt <= X86_XCPT_LAST);
776	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
777	}
778
779
780	/**
781	* Remove one or more exceptions from the exception bitmap and commits it to the
782	* current VMCS.
783	*
784	* This takes care of not removing the exception intercept if a nested-guest
785	* requires the exception to be intercepted.
786	*
787	* @returns VBox status code.
788	* @param pVCpu The cross context virtual CPU structure.
789	* @param pVmxTransient The VMX-transient structure.
790	* @param uXcptMask The exception(s) to remove.
791	*/
792	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
793	{
794	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
795	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
796	if (u32XcptBitmap & uXcptMask)
797	{
798	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
799	if (!pVmxTransient->fIsNestedGuest)
800	{ /* likely */ }
801	else
802	{
803	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
804	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
805	}
806	#endif
807	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
808	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
809	\| RT_BIT(X86_XCPT_DE)
810	\| RT_BIT(X86_XCPT_NM)
811	\| RT_BIT(X86_XCPT_TS)
812	\| RT_BIT(X86_XCPT_UD)
813	\| RT_BIT(X86_XCPT_NP)
814	\| RT_BIT(X86_XCPT_SS)
815	\| RT_BIT(X86_XCPT_GP)
816	\| RT_BIT(X86_XCPT_PF)
817	\| RT_BIT(X86_XCPT_MF));
818	#elif defined(HMVMX_ALWAYS_TRAP_PF)
819	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
820	#endif
821	if (uXcptMask)
822	{
823	/* Validate we are not removing any essential exception intercepts. */
824	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
825	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
826	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
827
828	/* Remove it from the exception bitmap. */
829	u32XcptBitmap &= ~uXcptMask;
830
831	/* Commit and update the cache if necessary. */
832	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
833	{
834	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
835	AssertRCReturn(rc, rc);
836	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
837	}
838	}
839	}
840	return VINF_SUCCESS;
841	}
842
843
844	/**
845	* Remove an exceptions from the exception bitmap and commits it to the current
846	* VMCS.
847	*
848	* @returns VBox status code.
849	* @param pVCpu The cross context virtual CPU structure.
850	* @param pVmxTransient The VMX-transient structure.
851	* @param uXcpt The exception to remove.
852	*/
853	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
854	{
855	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
856	}
857
858
859	/**
860	* Loads the VMCS specified by the VMCS info. object.
861	*
862	* @returns VBox status code.
863	* @param pVmcsInfo The VMCS info. object.
864	*/
865	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
866	{
867	Assert(pVmcsInfo);
868	Assert(pVmcsInfo->HCPhysVmcs);
869	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
870
871	if (pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_CLEAR)
872	{
873	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
874	if (RT_SUCCESS(rc))
875	{
876	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
877	return VINF_SUCCESS;
878	}
879	return rc;
880	}
881	return VERR_VMX_INVALID_VMCS_LAUNCH_STATE;
882	}
883
884
885	/**
886	* Clears the VMCS specified by the VMCS info. object.
887	*
888	* @returns VBox status code.
889	* @param pVmcsInfo The VMCS info. object.
890	*/
891	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
892	{
893	Assert(pVmcsInfo);
894	Assert(pVmcsInfo->HCPhysVmcs);
895	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
896
897	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
898	if (RT_SUCCESS(rc))
899	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
900	return rc;
901	}
902
903
904	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
905	/**
906	* Switches the current VMCS to the one specified.
907	*
908	* @returns VBox status code.
909	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
910	* @param pVmcsInfoTo The VMCS info. object we are switching to.
911	*
912	* @remarks Called with interrupts disabled.
913	*/
914	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
915	{
916	Assert(pVmcsInfoFrom);
917	Assert(pVmcsInfoTo);
918
919	/*
920	* Clear the VMCS we are switching out if it has not already been cleared.
921	* This will sync any CPU internal data back to the VMCS.
922	*/
923	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
924	{
925	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
926	if (RT_SUCCESS(rc))
927	{ /* likely */ }
928	else
929	return rc;
930	}
931
932	/*
933	* Clear the VMCS we are switching to if it has not already been cleared.
934	* This will initialize the VMCS launch state to "clear" required for loading it.
935	*
936	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
937	*/
938	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
939	{
940	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
941	if (RT_SUCCESS(rc))
942	{ /* likely */ }
943	else
944	return rc;
945	}
946
947	/*
948	* Finally, load the VMCS we are switching to.
949	*/
950	return hmR0VmxLoadVmcs(pVmcsInfoTo);
951	}
952	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
953
954
955	/**
956	* Updates the VM's last error record.
957	*
958	* If there was a VMX instruction error, reads the error data from the VMCS and
959	* updates VCPU's last error record as well.
960	*
961	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
962	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
963	* VERR_VMX_INVALID_VMCS_FIELD.
964	* @param rc The error code.
965	*/
966	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
967	{
968	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
969	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
970	{
971	AssertPtrReturnVoid(pVCpu);
972	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
973	}
974	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
975	}
976
977
978	#ifdef VBOX_STRICT
979	/**
980	* Reads the VM-entry interruption-information field from the VMCS into the VMX
981	* transient structure.
982	*
983	* @returns VBox status code.
984	* @param pVmxTransient The VMX-transient structure.
985	*
986	* @remarks No-long-jump zone!!!
987	*/
988	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
989	{
990	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
991	AssertRCReturn(rc, rc);
992	return VINF_SUCCESS;
993	}
994
995
996	/**
997	* Reads the VM-entry exception error code field from the VMCS into
998	* the VMX transient structure.
999	*
1000	* @returns VBox status code.
1001	* @param pVmxTransient The VMX-transient structure.
1002	*
1003	* @remarks No-long-jump zone!!!
1004	*/
1005	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1006	{
1007	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1008	AssertRCReturn(rc, rc);
1009	return VINF_SUCCESS;
1010	}
1011
1012
1013	/**
1014	* Reads the VM-entry exception error code field from the VMCS into
1015	* the VMX transient structure.
1016	*
1017	* @returns VBox status code.
1018	* @param pVmxTransient The VMX-transient structure.
1019	*
1020	* @remarks No-long-jump zone!!!
1021	*/
1022	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1023	{
1024	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1025	AssertRCReturn(rc, rc);
1026	return VINF_SUCCESS;
1027	}
1028	#endif /* VBOX_STRICT */
1029
1030
1031	/**
1032	* Reads the VM-exit interruption-information field from the VMCS into the VMX
1033	* transient structure.
1034	*
1035	* @returns VBox status code.
1036	* @param pVmxTransient The VMX-transient structure.
1037	*/
1038	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1039	{
1040	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1041	{
1042	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1043	AssertRCReturn(rc,rc);
1044	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
1045	}
1046	return VINF_SUCCESS;
1047	}
1048
1049
1050	/**
1051	* Reads the VM-exit interruption error code from the VMCS into the VMX
1052	* transient structure.
1053	*
1054	* @returns VBox status code.
1055	* @param pVmxTransient The VMX-transient structure.
1056	*/
1057	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1058	{
1059	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1060	{
1061	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1062	AssertRCReturn(rc, rc);
1063	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1064	}
1065	return VINF_SUCCESS;
1066	}
1067
1068
1069	/**
1070	* Reads the VM-exit instruction length field from the VMCS into the VMX
1071	* transient structure.
1072	*
1073	* @returns VBox status code.
1074	* @param pVmxTransient The VMX-transient structure.
1075	*/
1076	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1077	{
1078	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1079	{
1080	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1081	AssertRCReturn(rc, rc);
1082	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1083	}
1084	return VINF_SUCCESS;
1085	}
1086
1087
1088	/**
1089	* Reads the VM-exit instruction-information field from the VMCS into
1090	* the VMX transient structure.
1091	*
1092	* @returns VBox status code.
1093	* @param pVmxTransient The VMX-transient structure.
1094	*/
1095	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1096	{
1097	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1098	{
1099	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1100	AssertRCReturn(rc, rc);
1101	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1102	}
1103	return VINF_SUCCESS;
1104	}
1105
1106
1107	/**
1108	* Reads the VM-exit Qualification from the VMCS into the VMX transient structure.
1109	*
1110	* @returns VBox status code.
1111	* @param pVCpu The cross context virtual CPU structure of the
1112	* calling EMT. (Required for the VMCS cache case.)
1113	* @param pVmxTransient The VMX-transient structure.
1114	*/
1115	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1116	{
1117	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1118	{
1119	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1120	AssertRCReturn(rc, rc);
1121	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1122	}
1123	return VINF_SUCCESS;
1124	}
1125
1126
1127	/**
1128	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1129	*
1130	* @returns VBox status code.
1131	* @param pVCpu The cross context virtual CPU structure of the
1132	* calling EMT. (Required for the VMCS cache case.)
1133	* @param pVmxTransient The VMX-transient structure.
1134	*/
1135	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1136	{
1137	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1138	{
1139	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1140	AssertRCReturn(rc, rc);
1141	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1142	}
1143	return VINF_SUCCESS;
1144	}
1145
1146
1147	/**
1148	* Reads the IDT-vectoring information field from the VMCS into the VMX
1149	* transient structure.
1150	*
1151	* @returns VBox status code.
1152	* @param pVmxTransient The VMX-transient structure.
1153	*
1154	* @remarks No-long-jump zone!!!
1155	*/
1156	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1157	{
1158	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1159	{
1160	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1161	AssertRCReturn(rc, rc);
1162	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1163	}
1164	return VINF_SUCCESS;
1165	}
1166
1167
1168	/**
1169	* Reads the IDT-vectoring error code from the VMCS into the VMX
1170	* transient structure.
1171	*
1172	* @returns VBox status code.
1173	* @param pVmxTransient The VMX-transient structure.
1174	*/
1175	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1176	{
1177	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1178	{
1179	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1180	AssertRCReturn(rc, rc);
1181	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1182	}
1183	return VINF_SUCCESS;
1184	}
1185
1186
1187	/**
1188	* Enters VMX root mode operation on the current CPU.
1189	*
1190	* @returns VBox status code.
1191	* @param pVM The cross context VM structure. Can be
1192	* NULL, after a resume.
1193	* @param HCPhysCpuPage Physical address of the VMXON region.
1194	* @param pvCpuPage Pointer to the VMXON region.
1195	*/
1196	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1197	{
1198	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1199	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1200	Assert(pvCpuPage);
1201	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1202
1203	if (pVM)
1204	{
1205	/* Write the VMCS revision identifier to the VMXON region. */
1206	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1207	}
1208
1209	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1210	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1211
1212	/* Enable the VMX bit in CR4 if necessary. */
1213	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1214
1215	/* Enter VMX root mode. */
1216	int rc = VMXEnable(HCPhysCpuPage);
1217	if (RT_FAILURE(rc))
1218	{
1219	if (!(uOldCr4 & X86_CR4_VMXE))
1220	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1221
1222	if (pVM)
1223	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1224	}
1225
1226	/* Restore interrupts. */
1227	ASMSetFlags(fEFlags);
1228	return rc;
1229	}
1230
1231
1232	/**
1233	* Exits VMX root mode operation on the current CPU.
1234	*
1235	* @returns VBox status code.
1236	*/
1237	static int hmR0VmxLeaveRootMode(void)
1238	{
1239	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1240
1241	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1242	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1243
1244	/* If we're for some reason not in VMX root mode, then don't leave it. */
1245	RTCCUINTREG const uHostCR4 = ASMGetCR4();
1246
1247	int rc;
1248	if (uHostCR4 & X86_CR4_VMXE)
1249	{
1250	/* Exit VMX root mode and clear the VMX bit in CR4. */
1251	VMXDisable();
1252	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1253	rc = VINF_SUCCESS;
1254	}
1255	else
1256	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1257
1258	/* Restore interrupts. */
1259	ASMSetFlags(fEFlags);
1260	return rc;
1261	}
1262
1263
1264	/**
1265	* Allocates and maps a physically contiguous page. The allocated page is
1266	* zero'd out (used by various VT-x structures).
1267	*
1268	* @returns IPRT status code.
1269	* @param pMemObj Pointer to the ring-0 memory object.
1270	* @param ppVirt Where to store the virtual address of the
1271	* allocation.
1272	* @param pHCPhys Where to store the physical address of the
1273	* allocation.
1274	*/
1275	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1276	{
1277	AssertPtr(pMemObj);
1278	AssertPtr(ppVirt);
1279	AssertPtr(pHCPhys);
1280	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1281	if (RT_FAILURE(rc))
1282	return rc;
1283	ppVirt = RTR0MemObjAddress(pMemObj);
1284	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1285	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1286	return VINF_SUCCESS;
1287	}
1288
1289
1290	/**
1291	* Frees and unmaps an allocated, physical page.
1292	*
1293	* @param pMemObj Pointer to the ring-0 memory object.
1294	* @param ppVirt Where to re-initialize the virtual address of
1295	* allocation as 0.
1296	* @param pHCPhys Where to re-initialize the physical address of the
1297	* allocation as 0.
1298	*/
1299	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1300	{
1301	AssertPtr(pMemObj);
1302	AssertPtr(ppVirt);
1303	AssertPtr(pHCPhys);
1304	/* NULL is valid, accepted and ignored by the free function below. */
1305	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1306	*pMemObj = NIL_RTR0MEMOBJ;
1307	*ppVirt = NULL;
1308	*pHCPhys = NIL_RTHCPHYS;
1309	}
1310
1311
1312	/**
1313	* Initializes a VMCS info. object.
1314	*
1315	* @param pVmcsInfo The VMCS info. object.
1316	*/
1317	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1318	{
1319	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1320
1321	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1322	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1323	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1324	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1325	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1326	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1327	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1328	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1329	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1330	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1331	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1332	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1333	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1334	}
1335
1336
1337	/**
1338	* Frees the VT-x structures for a VMCS info. object.
1339	*
1340	* @param pVM The cross context VM structure.
1341	* @param pVmcsInfo The VMCS info. object.
1342	*/
1343	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1344	{
1345	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1346
1347	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1348	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1349
1350	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1351	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1352	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1353
1354	hmR0VmxInitVmcsInfo(pVmcsInfo);
1355	}
1356
1357
1358	/**
1359	* Allocates the VT-x structures for a VMCS info. object.
1360	*
1361	* @returns VBox status code.
1362	* @param pVCpu The cross context virtual CPU structure.
1363	* @param pVmcsInfo The VMCS info. object.
1364	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1365	*/
1366	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1367	{
1368	PVM pVM = pVCpu->CTX_SUFF(pVM);
1369
1370	/* Allocate the guest VM control structure (VMCS). */
1371	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1372	if (RT_SUCCESS(rc))
1373	{
1374	if (!fIsNstGstVmcs)
1375	{
1376	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1377	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1378	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1379	{
1380	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1381	NULL /* pR3Ptr /, NULL / pRCPtr */);
1382	}
1383	}
1384	else
1385	{
1386	Assert(pVmcsInfo->HCPhysVirtApic == NIL_RTHCPHYS);
1387	Assert(!pVmcsInfo->pbVirtApic);
1388	}
1389
1390	if (RT_SUCCESS(rc))
1391	{
1392	/*
1393	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1394	* transparent accesses of specific MSRs.
1395	*
1396	* If the condition for enabling MSR bitmaps changes here, don't forget to
1397	* update HMIsMsrBitmapActive().
1398	*
1399	* We don't share MSR bitmaps between the guest and nested-guest as we then
1400	* don't need to care about carefully restoring the guest MSR bitmap.
1401	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1402	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1403	*/
1404	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1405	{
1406	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1407	if (RT_SUCCESS(rc))
1408	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1409	}
1410
1411	if (RT_SUCCESS(rc))
1412	{
1413	/*
1414	* Allocate the VM-entry MSR-load area for the guest MSRs.
1415	*
1416	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1417	* the guest and nested-guest.
1418	*/
1419	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1420	&pVmcsInfo->HCPhysGuestMsrLoad);
1421	if (RT_SUCCESS(rc))
1422	{
1423	/*
1424	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1425	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1426	*/
1427	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1428	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1429	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1430
1431	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1432	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1433	&pVmcsInfo->HCPhysHostMsrLoad);
1434	}
1435	}
1436	}
1437	}
1438
1439	return rc;
1440	}
1441
1442
1443	/**
1444	* Free all VT-x structures for the VM.
1445	*
1446	* @returns IPRT status code.
1447	* @param pVM The cross context VM structure.
1448	*/
1449	static void hmR0VmxStructsFree(PVM pVM)
1450	{
1451	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1452	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1453	#endif
1454	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1455
1456	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1457	{
1458	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1459	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1460	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1461	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1462	if (pVM->cpum.ro.GuestFeatures.fVmx)
1463	{
1464	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1465	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1466	}
1467	#endif
1468	}
1469	}
1470
1471
1472	/**
1473	* Allocate all VT-x structures for the VM.
1474	*
1475	* @returns IPRT status code.
1476	* @param pVM The cross context VM structure.
1477	*/
1478	static int hmR0VmxStructsAlloc(PVM pVM)
1479	{
1480	/*
1481	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1482	* The VMCS size cannot be more than 4096 bytes.
1483	*
1484	* See Intel spec. Appendix A.1 "Basic VMX Information".
1485	*/
1486	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1487	if (cbVmcs <= X86_PAGE_4K_SIZE)
1488	{ /* likely */ }
1489	else
1490	{
1491	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1492	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1493	}
1494
1495	/*
1496	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1497	*/
1498	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1499	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1500	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1501	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1502	#endif
1503
1504	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1505	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1506	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1507
1508	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1509	{
1510	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1511	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1512	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1513	}
1514
1515	/*
1516	* Allocate per-VM VT-x structures.
1517	*/
1518	int rc = VINF_SUCCESS;
1519	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1520	/* Allocate crash-dump magic scratch page. */
1521	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1522	if (RT_FAILURE(rc))
1523	{
1524	hmR0VmxStructsFree(pVM);
1525	return rc;
1526	}
1527	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1528	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1529	#endif
1530
1531	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1532	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1533	{
1534	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1535	&pVM->hm.s.vmx.HCPhysApicAccess);
1536	if (RT_FAILURE(rc))
1537	{
1538	hmR0VmxStructsFree(pVM);
1539	return rc;
1540	}
1541	}
1542
1543	/*
1544	* Initialize per-VCPU VT-x structures.
1545	*/
1546	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1547	{
1548	/* Allocate the guest VMCS structures. */
1549	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1550	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
1551	if (RT_SUCCESS(rc))
1552	{
1553	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1554	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
1555	if (pVM->cpum.ro.GuestFeatures.fVmx)
1556	{
1557	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
1558	if (RT_SUCCESS(rc))
1559	{ /* likely */ }
1560	else
1561	break;
1562	}
1563	#endif
1564	}
1565	else
1566	break;
1567	}
1568
1569	if (RT_FAILURE(rc))
1570	{
1571	hmR0VmxStructsFree(pVM);
1572	return rc;
1573	}
1574
1575	return VINF_SUCCESS;
1576	}
1577
1578
1579	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1580	/**
1581	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
1582	*
1583	* @returns @c true if the MSR is intercepted, @c false otherwise.
1584	* @param pvMsrBitmap The MSR bitmap.
1585	* @param offMsr The MSR byte offset.
1586	* @param iBit The bit offset from the byte offset.
1587	*/
1588	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
1589	{
1590	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
1591	Assert(pbMsrBitmap);
1592	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
1593	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
1594	}
1595	#endif
1596
1597
1598	/**
1599	* Sets the permission bits for the specified MSR in the given MSR bitmap.
1600	*
1601	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
1602	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
1603	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
1604	* the read/write access of this MSR.
1605	*
1606	* @param pVCpu The cross context virtual CPU structure.
1607	* @param pVmcsInfo The VMCS info. object.
1608	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1609	* @param idMsr The MSR value.
1610	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
1611	* include both a read -and- a write permission!
1612	*
1613	* @sa CPUMGetVmxMsrPermission.
1614	* @remarks Can be called with interrupts disabled.
1615	*/
1616	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
1617	{
1618	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
1619	Assert(pbMsrBitmap);
1620	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
1621
1622	/*
1623	* MSR-bitmap Layout:
1624	* Byte index MSR range Interpreted as
1625	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1626	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1627	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1628	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1629	*
1630	* A bit corresponding to an MSR within the above range causes a VM-exit
1631	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
1632	* the MSR range, it always cause a VM-exit.
1633	*
1634	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
1635	*/
1636	uint16_t const offBitmapRead = 0;
1637	uint16_t const offBitmapWrite = 0x800;
1638	uint16_t offMsr;
1639	int32_t iBit;
1640	if (idMsr <= UINT32_C(0x00001fff))
1641	{
1642	offMsr = 0;
1643	iBit = idMsr;
1644	}
1645	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1646	{
1647	offMsr = 0x400;
1648	iBit = idMsr - UINT32_C(0xc0000000);
1649	}
1650	else
1651	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
1652
1653	/*
1654	* Set the MSR read permission.
1655	*/
1656	uint16_t const offMsrRead = offBitmapRead + offMsr;
1657	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
1658	if (fMsrpm & VMXMSRPM_ALLOW_RD)
1659	{
1660	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1661	bool const fClear = !fIsNstGstVmcs ? true
1662	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
1663	#else
1664	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1665	bool const fClear = true;
1666	#endif
1667	if (fClear)
1668	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
1669	}
1670	else
1671	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
1672
1673	/*
1674	* Set the MSR write permission.
1675	*/
1676	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
1677	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
1678	if (fMsrpm & VMXMSRPM_ALLOW_WR)
1679	{
1680	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1681	bool const fClear = !fIsNstGstVmcs ? true
1682	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
1683	#else
1684	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1685	bool const fClear = true;
1686	#endif
1687	if (fClear)
1688	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
1689	}
1690	else
1691	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
1692	}
1693
1694
1695	/**
1696	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1697	* area.
1698	*
1699	* @returns VBox status code.
1700	* @param pVCpu The cross context virtual CPU structure.
1701	* @param pVmcsInfo The VMCS info. object.
1702	* @param cMsrs The number of MSRs.
1703	*/
1704	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
1705	{
1706	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1707	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1708	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
1709	{
1710	/* Commit the MSR counts to the VMCS and update the cache. */
1711	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
1712	{
1713	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1714	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1715	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1716	AssertRCReturn(rc, rc);
1717
1718	pVmcsInfo->cEntryMsrLoad = cMsrs;
1719	pVmcsInfo->cExitMsrStore = cMsrs;
1720	pVmcsInfo->cExitMsrLoad = cMsrs;
1721	}
1722	return VINF_SUCCESS;
1723	}
1724
1725	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
1726	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1727	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1728	}
1729
1730
1731	/**
1732	* Adds a new (or updates the value of an existing) guest/host MSR
1733	* pair to be swapped during the world-switch as part of the
1734	* auto-load/store MSR area in the VMCS.
1735	*
1736	* @returns VBox status code.
1737	* @param pVCpu The cross context virtual CPU structure.
1738	* @param pVmxTransient The VMX-transient structure.
1739	* @param idMsr The MSR.
1740	* @param uGuestMsrValue Value of the guest MSR.
1741	* @param fSetReadWrite Whether to set the guest read/write access of this
1742	* MSR (thus not causing a VM-exit).
1743	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1744	* necessary.
1745	*/
1746	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
1747	bool fSetReadWrite, bool fUpdateHostMsr)
1748	{
1749	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1750	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1751	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1752	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1753	uint32_t i;
1754
1755	/* Paranoia. */
1756	Assert(pGuestMsrLoad);
1757
1758	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
1759
1760	/* Check if the MSR already exists in the VM-entry MSR-load area. */
1761	for (i = 0; i < cMsrs; i++)
1762	{
1763	if (pGuestMsrLoad[i].u32Msr == idMsr)
1764	break;
1765	}
1766
1767	bool fAdded = false;
1768	if (i == cMsrs)
1769	{
1770	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
1771	++cMsrs;
1772	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1773	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
1774
1775	/* Set the guest to read/write this MSR without causing VM-exits. */
1776	if ( fSetReadWrite
1777	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
1778	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
1779
1780	LogFlowFunc(("MSR added, cMsrs now %u\n", cMsrs));
1781	fAdded = true;
1782	}
1783
1784	/* Update the MSR value for the newly added or already existing MSR. */
1785	pGuestMsrLoad[i].u32Msr = idMsr;
1786	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
1787
1788	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
1789	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1790	{
1791	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1792	pGuestMsrStore[i].u32Msr = idMsr;
1793	pGuestMsrStore[i].u64Value = uGuestMsrValue;
1794	}
1795
1796	/* Update the corresponding slot in the host MSR area. */
1797	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1798	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
1799	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
1800	pHostMsr[i].u32Msr = idMsr;
1801
1802	/*
1803	* Only if the caller requests to update the host MSR value AND we've newly added the
1804	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
1805	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
1806	*
1807	* We do this for performance reasons since reading MSRs may be quite expensive.
1808	*/
1809	if (fAdded)
1810	{
1811	if (fUpdateHostMsr)
1812	{
1813	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1814	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1815	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
1816	}
1817	else
1818	{
1819	/* Someone else can do the work. */
1820	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
1821	}
1822	}
1823	return VINF_SUCCESS;
1824	}
1825
1826
1827	/**
1828	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1829	* auto-load/store MSR area in the VMCS.
1830	*
1831	* @returns VBox status code.
1832	* @param pVCpu The cross context virtual CPU structure.
1833	* @param pVmxTransient The VMX-transient structure.
1834	* @param idMsr The MSR.
1835	*/
1836	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
1837	{
1838	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1839	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1840	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1841	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1842
1843	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
1844
1845	for (uint32_t i = 0; i < cMsrs; i++)
1846	{
1847	/* Find the MSR. */
1848	if (pGuestMsrLoad[i].u32Msr == idMsr)
1849	{
1850	/*
1851	* If it's the last MSR, we only need to reduce the MSR count.
1852	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
1853	*/
1854	if (i < cMsrs - 1)
1855	{
1856	/* Remove it from the VM-entry MSR-load area. */
1857	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
1858	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
1859
1860	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
1861	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1862	{
1863	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1864	Assert(pGuestMsrStore[i].u32Msr == idMsr);
1865	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
1866	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
1867	}
1868
1869	/* Remove it from the VM-exit MSR-load area. */
1870	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1871	Assert(pHostMsr[i].u32Msr == idMsr);
1872	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
1873	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
1874	}
1875
1876	/* Reduce the count to reflect the removed MSR and bail. */
1877	--cMsrs;
1878	break;
1879	}
1880	}
1881
1882	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
1883	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
1884	{
1885	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1886	AssertRCReturn(rc, rc);
1887
1888	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1889	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1890	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
1891
1892	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
1893	return VINF_SUCCESS;
1894	}
1895
1896	return VERR_NOT_FOUND;
1897	}
1898
1899
1900	/**
1901	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
1902	*
1903	* @returns @c true if found, @c false otherwise.
1904	* @param pVmcsInfo The VMCS info. object.
1905	* @param idMsr The MSR to find.
1906	*/
1907	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
1908	{
1909	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1910	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
1911	Assert(pMsrs);
1912	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
1913	for (uint32_t i = 0; i < cMsrs; i++)
1914	{
1915	if (pMsrs[i].u32Msr == idMsr)
1916	return true;
1917	}
1918	return false;
1919	}
1920
1921
1922	/**
1923	* Updates the value of all host MSRs in the VM-exit MSR-load area.
1924	*
1925	* @param pVCpu The cross context virtual CPU structure.
1926	* @param pVmcsInfo The VMCS info. object.
1927	*
1928	* @remarks No-long-jump zone!!!
1929	*/
1930	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
1931	{
1932	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1933
1934	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1935	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
1936	Assert(pHostMsrLoad);
1937	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
1938	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
1939	for (uint32_t i = 0; i < cMsrs; i++)
1940	{
1941	/*
1942	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1943	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1944	*/
1945	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
1946	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
1947	else
1948	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
1949	}
1950	}
1951
1952
1953	/**
1954	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1955	* perform lazy restoration of the host MSRs while leaving VT-x.
1956	*
1957	* @param pVCpu The cross context virtual CPU structure.
1958	*
1959	* @remarks No-long-jump zone!!!
1960	*/
1961	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1962	{
1963	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1964
1965	/*
1966	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
1967	*/
1968	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1969	{
1970	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1971	#if HC_ARCH_BITS == 64
1972	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1973	{
1974	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
1975	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
1976	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
1977	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1978	}
1979	#endif
1980	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1981	}
1982	}
1983
1984
1985	/**
1986	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1987	* lazily while leaving VT-x.
1988	*
1989	* @returns true if it does, false otherwise.
1990	* @param pVCpu The cross context virtual CPU structure.
1991	* @param idMsr The MSR to check.
1992	*/
1993	static bool hmR0VmxIsLazyGuestMsr(PCVMCPU pVCpu, uint32_t idMsr)
1994	{
1995	NOREF(pVCpu);
1996	#if HC_ARCH_BITS == 64
1997	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1998	{
1999	switch (idMsr)
2000	{
2001	case MSR_K8_LSTAR:
2002	case MSR_K6_STAR:
2003	case MSR_K8_SF_MASK:
2004	case MSR_K8_KERNEL_GS_BASE:
2005	return true;
2006	}
2007	}
2008	#else
2009	RT_NOREF(pVCpu, idMsr);
2010	#endif
2011	return false;
2012	}
2013
2014
2015	/**
2016	* Loads a set of guests MSRs to allow read/passthru to the guest.
2017	*
2018	* The name of this function is slightly confusing. This function does NOT
2019	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
2020	* common prefix for functions dealing with "lazy restoration" of the shared
2021	* MSRs.
2022	*
2023	* @param pVCpu The cross context virtual CPU structure.
2024	*
2025	* @remarks No-long-jump zone!!!
2026	*/
2027	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
2028	{
2029	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2030	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2031
2032	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2033	#if HC_ARCH_BITS == 64
2034	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2035	{
2036	/*
2037	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
2038	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
2039	* we can skip a few MSR writes.
2040	*
2041	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
2042	* guest MSR values in the guest-CPU context might be different to what's currently
2043	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
2044	* CPU, see @bugref{8728}.
2045	*/
2046	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2047	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2048	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
2049	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
2050	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
2051	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
2052	{
2053	#ifdef VBOX_STRICT
2054	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
2055	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
2056	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
2057	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
2058	#endif
2059	}
2060	else
2061	{
2062	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2063	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2064	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2065	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2066	}
2067	}
2068	#endif
2069	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2070	}
2071
2072
2073	/**
2074	* Performs lazy restoration of the set of host MSRs if they were previously
2075	* loaded with guest MSR values.
2076	*
2077	* @param pVCpu The cross context virtual CPU structure.
2078	*
2079	* @remarks No-long-jump zone!!!
2080	* @remarks The guest MSRs should have been saved back into the guest-CPU
2081	* context by hmR0VmxImportGuestState()!!!
2082	*/
2083	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2084	{
2085	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2086	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2087
2088	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2089	{
2090	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2091	#if HC_ARCH_BITS == 64
2092	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2093	{
2094	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2095	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2096	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2097	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2098	}
2099	#endif
2100	}
2101	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2102	}
2103
2104
2105	/**
2106	* Verifies that our cached values of the VMCS fields are all consistent with
2107	* what's actually present in the VMCS.
2108	*
2109	* @returns VBox status code.
2110	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2111	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2112	* VMCS content. HMCPU error-field is
2113	* updated, see VMX_VCI_XXX.
2114	* @param pVCpu The cross context virtual CPU structure.
2115	* @param pVmcsInfo The VMCS info. object.
2116	*/
2117	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2118	{
2119	uint32_t u32Val;
2120	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2121	AssertRCReturn(rc, rc);
2122	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2123	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32EntryCtls, u32Val),
2124	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2125	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2126
2127	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2128	AssertRCReturn(rc, rc);
2129	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2130	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ExitCtls, u32Val),
2131	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2132	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2133
2134	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2135	AssertRCReturn(rc, rc);
2136	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2137	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32PinCtls, u32Val),
2138	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2139	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2140
2141	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2142	AssertRCReturn(rc, rc);
2143	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2144	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls, u32Val),
2145	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2146	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2147
2148	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2149	{
2150	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2151	AssertRCReturn(rc, rc);
2152	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2153	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls2, u32Val),
2154	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2155	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2156	}
2157
2158	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2159	AssertRCReturn(rc, rc);
2160	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2161	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32XcptBitmap, u32Val),
2162	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2163	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2164
2165	uint64_t u64Val;
2166	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2167	AssertRCReturn(rc, rc);
2168	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2169	("Cache=%#RX64 VMCS=%#RX64\n", pVmcsInfo->u64TscOffset, u64Val),
2170	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2171	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2172
2173	return VINF_SUCCESS;
2174	}
2175
2176
2177	#ifdef VBOX_STRICT
2178	/**
2179	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2180	*
2181	* @param pVCpu The cross context virtual CPU structure.
2182	* @param pVmcsInfo The VMCS info. object.
2183	*/
2184	static void hmR0VmxCheckHostEferMsr(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2185	{
2186	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2187
2188	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2189	{
2190	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2191	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2192	uint64_t uVmcsEferMsrVmcs;
2193	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2194	AssertRC(rc);
2195
2196	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2197	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2198	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2199	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2200	}
2201	}
2202
2203
2204	/**
2205	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2206	* VMCS are correct.
2207	*
2208	* @param pVCpu The cross context virtual CPU structure.
2209	* @param pVmcsInfo The VMCS info. object.
2210	*/
2211	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2212	{
2213	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2214
2215	/* Read the various MSR-area counts from the VMCS. */
2216	uint32_t cEntryLoadMsrs;
2217	uint32_t cExitStoreMsrs;
2218	uint32_t cExitLoadMsrs;
2219	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2220	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2221	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2222
2223	/* Verify all the MSR counts are the same. */
2224	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2225	Assert(cExitStoreMsrs == cExitLoadMsrs);
2226	uint32_t const cMsrs = cExitLoadMsrs;
2227
2228	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2229	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2230
2231	/* Verify the MSR counts are within the allocated page size. */
2232	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2233
2234	/* Verify the relevant contents of the MSR areas match. */
2235	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2236	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2237	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2238	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2239	for (uint32_t i = 0; i < cMsrs; i++)
2240	{
2241	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2242	if (fSeparateExitMsrStorePage)
2243	{
2244	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2245	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2246	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2247	}
2248
2249	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2250	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2251	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2252
2253	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2254	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2255	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2256	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2257
2258	/* Verify that cached host EFER MSR matches what's loaded the CPU. */
2259	bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2260	if (fIsEferMsr)
2261	{
2262	AssertMsgReturnVoid(u64Msr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2263	("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2264	pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64Msr, cMsrs));
2265	}
2266
2267	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2268	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2269	{
2270	uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2271	if (fIsEferMsr)
2272	{
2273	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2274	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2275	}
2276	else
2277	{
2278	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2279	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2280	}
2281	}
2282
2283	/* Move to the next MSR. */
2284	pHostMsrLoad++;
2285	pGuestMsrLoad++;
2286	pGuestMsrStore++;
2287	}
2288	}
2289	#endif /* VBOX_STRICT */
2290
2291
2292	/**
2293	* Flushes the TLB using EPT.
2294	*
2295	* @returns VBox status code.
2296	* @param pVCpu The cross context virtual CPU structure of the calling
2297	* EMT. Can be NULL depending on @a enmTlbFlush.
2298	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2299	* enmTlbFlush.
2300	* @param enmTlbFlush Type of flush.
2301	*
2302	* @remarks Caller is responsible for making sure this function is called only
2303	* when NestedPaging is supported and providing @a enmTlbFlush that is
2304	* supported by the CPU.
2305	* @remarks Can be called with interrupts disabled.
2306	*/
2307	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2308	{
2309	uint64_t au64Descriptor[2];
2310	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2311	au64Descriptor[0] = 0;
2312	else
2313	{
2314	Assert(pVCpu);
2315	Assert(pVmcsInfo);
2316	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2317	}
2318	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2319
2320	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2321	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2322
2323	if ( RT_SUCCESS(rc)
2324	&& pVCpu)
2325	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2326	}
2327
2328
2329	/**
2330	* Flushes the TLB using VPID.
2331	*
2332	* @returns VBox status code.
2333	* @param pVCpu The cross context virtual CPU structure of the calling
2334	* EMT. Can be NULL depending on @a enmTlbFlush.
2335	* @param enmTlbFlush Type of flush.
2336	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2337	* on @a enmTlbFlush).
2338	*
2339	* @remarks Can be called with interrupts disabled.
2340	*/
2341	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2342	{
2343	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2344
2345	uint64_t au64Descriptor[2];
2346	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2347	{
2348	au64Descriptor[0] = 0;
2349	au64Descriptor[1] = 0;
2350	}
2351	else
2352	{
2353	AssertPtr(pVCpu);
2354	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2355	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2356	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2357	au64Descriptor[1] = GCPtr;
2358	}
2359
2360	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2361	AssertMsg(rc == VINF_SUCCESS,
2362	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2363
2364	if ( RT_SUCCESS(rc)
2365	&& pVCpu)
2366	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2367	NOREF(rc);
2368	}
2369
2370
2371	/**
2372	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2373	* otherwise there is nothing really to invalidate.
2374	*
2375	* @returns VBox status code.
2376	* @param pVCpu The cross context virtual CPU structure.
2377	* @param GCVirt Guest virtual address of the page to invalidate.
2378	*/
2379	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2380	{
2381	AssertPtr(pVCpu);
2382	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2383
2384	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2385	{
2386	/*
2387	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2388	* the EPT case. See @bugref{6043} and @bugref{6177}.
2389	*
2390	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2391	* as this function maybe called in a loop with individual addresses.
2392	*/
2393	PVM pVM = pVCpu->CTX_SUFF(pVM);
2394	if (pVM->hm.s.vmx.fVpid)
2395	{
2396	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2397
2398	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2399	/*
2400	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2401	* where executing INVVPID outside 64-bit mode does not flush translations of
2402	* 64-bit linear addresses, see @bugref{6208#c72}.
2403	*/
2404	if (RT_HI_U32(GCVirt))
2405	fVpidFlush = false;
2406	#endif
2407
2408	if (fVpidFlush)
2409	{
2410	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2411	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2412	}
2413	else
2414	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2415	}
2416	else if (pVM->hm.s.fNestedPaging)
2417	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2418	}
2419
2420	return VINF_SUCCESS;
2421	}
2422
2423
2424	/**
2425	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2426	* case where neither EPT nor VPID is supported by the CPU.
2427	*
2428	* @param pHostCpu The HM physical-CPU structure.
2429	* @param pVCpu The cross context virtual CPU structure.
2430	*
2431	* @remarks Called with interrupts disabled.
2432	*/
2433	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2434	{
2435	AssertPtr(pVCpu);
2436	AssertPtr(pHostCpu);
2437
2438	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2439
2440	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2441	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2442	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2443	pVCpu->hm.s.fForceTLBFlush = false;
2444	return;
2445	}
2446
2447
2448	/**
2449	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2450	*
2451	* @param pHostCpu The HM physical-CPU structure.
2452	* @param pVCpu The cross context virtual CPU structure.
2453	* @param pVmcsInfo The VMCS info. object.
2454	*
2455	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2456	* nomenclature. The reason is, to avoid confusion in compare statements
2457	* since the host-CPU copies are named "ASID".
2458	*
2459	* @remarks Called with interrupts disabled.
2460	*/
2461	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2462	{
2463	#ifdef VBOX_WITH_STATISTICS
2464	bool fTlbFlushed = false;
2465	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2466	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2467	if (!fTlbFlushed) \
2468	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2469	} while (0)
2470	#else
2471	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2472	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2473	#endif
2474
2475	AssertPtr(pVCpu);
2476	AssertPtr(pHostCpu);
2477	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2478
2479	PVM pVM = pVCpu->CTX_SUFF(pVM);
2480	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2481	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2482	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2483
2484	/*
2485	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2486	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2487	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2488	* cannot reuse the current ASID anymore.
2489	*/
2490	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2491	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2492	{
2493	++pHostCpu->uCurrentAsid;
2494	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2495	{
2496	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2497	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2498	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2499	}
2500
2501	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2502	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2503	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2504
2505	/*
2506	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2507	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2508	*/
2509	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2510	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2511	HMVMX_SET_TAGGED_TLB_FLUSHED();
2512	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2513	}
2514	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2515	{
2516	/*
2517	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2518	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2519	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2520	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2521	* mappings, see @bugref{6568}.
2522	*
2523	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2524	*/
2525	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2526	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2527	HMVMX_SET_TAGGED_TLB_FLUSHED();
2528	}
2529
2530	pVCpu->hm.s.fForceTLBFlush = false;
2531	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2532
2533	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
2534	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
2535	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2536	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2537	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2538	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2539	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2540	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2541	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2542
2543	/* Update VMCS with the VPID. */
2544	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2545	AssertRC(rc);
2546
2547	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2548	}
2549
2550
2551	/**
2552	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2553	*
2554	* @param pHostCpu The HM physical-CPU structure.
2555	* @param pVCpu The cross context virtual CPU structure.
2556	* @param pVmcsInfo The VMCS info. object.
2557	*
2558	* @remarks Called with interrupts disabled.
2559	*/
2560	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2561	{
2562	AssertPtr(pVCpu);
2563	AssertPtr(pHostCpu);
2564	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2565	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2566	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2567
2568	/*
2569	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2570	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2571	*/
2572	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2573	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2574	{
2575	pVCpu->hm.s.fForceTLBFlush = true;
2576	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2577	}
2578
2579	/* Check for explicit TLB flushes. */
2580	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2581	{
2582	pVCpu->hm.s.fForceTLBFlush = true;
2583	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2584	}
2585
2586	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2587	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2588
2589	if (pVCpu->hm.s.fForceTLBFlush)
2590	{
2591	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2592	pVCpu->hm.s.fForceTLBFlush = false;
2593	}
2594	}
2595
2596
2597	/**
2598	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2599	*
2600	* @param pHostCpu The HM physical-CPU structure.
2601	* @param pVCpu The cross context virtual CPU structure.
2602	*
2603	* @remarks Called with interrupts disabled.
2604	*/
2605	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2606	{
2607	AssertPtr(pVCpu);
2608	AssertPtr(pHostCpu);
2609	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2610	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2611	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2612
2613	/*
2614	* Force a TLB flush for the first world switch if the current CPU differs from the one we
2615	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2616	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2617	* cannot reuse the current ASID anymore.
2618	*/
2619	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2620	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2621	{
2622	pVCpu->hm.s.fForceTLBFlush = true;
2623	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2624	}
2625
2626	/* Check for explicit TLB flushes. */
2627	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2628	{
2629	/*
2630	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2631	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2632	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
2633	* include fExplicitFlush's too) - an obscure corner case.
2634	*/
2635	pVCpu->hm.s.fForceTLBFlush = true;
2636	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2637	}
2638
2639	PVM pVM = pVCpu->CTX_SUFF(pVM);
2640	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2641	if (pVCpu->hm.s.fForceTLBFlush)
2642	{
2643	++pHostCpu->uCurrentAsid;
2644	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2645	{
2646	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2647	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2648	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2649	}
2650
2651	pVCpu->hm.s.fForceTLBFlush = false;
2652	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2653	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2654	if (pHostCpu->fFlushAsidBeforeUse)
2655	{
2656	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2657	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2658	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2659	{
2660	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2661	pHostCpu->fFlushAsidBeforeUse = false;
2662	}
2663	else
2664	{
2665	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2666	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2667	}
2668	}
2669	}
2670
2671	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2672	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2673	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2674	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2675	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2676	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2677	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2678
2679	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2680	AssertRC(rc);
2681	}
2682
2683
2684	/**
2685	* Flushes the guest TLB entry based on CPU capabilities.
2686	*
2687	* @param pHostCpu The HM physical-CPU structure.
2688	* @param pVCpu The cross context virtual CPU structure.
2689	* @param pVmcsInfo The VMCS info. object.
2690	*
2691	* @remarks Called with interrupts disabled.
2692	*/
2693	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2694	{
2695	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2696	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2697	#endif
2698	PVM pVM = pVCpu->CTX_SUFF(pVM);
2699	switch (pVM->hm.s.vmx.enmTlbFlushType)
2700	{
2701	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
2702	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
2703	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
2704	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
2705	default:
2706	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2707	break;
2708	}
2709	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2710	}
2711
2712
2713	/**
2714	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2715	* TLB entries from the host TLB before VM-entry.
2716	*
2717	* @returns VBox status code.
2718	* @param pVM The cross context VM structure.
2719	*/
2720	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2721	{
2722	/*
2723	* Determine optimal flush type for nested paging.
2724	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
2725	* unrestricted guest execution (see hmR3InitFinalizeR0()).
2726	*/
2727	if (pVM->hm.s.fNestedPaging)
2728	{
2729	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2730	{
2731	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2732	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2733	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2734	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2735	else
2736	{
2737	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2738	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2739	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2740	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2741	}
2742
2743	/* Make sure the write-back cacheable memory type for EPT is supported. */
2744	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2745	{
2746	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2747	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2748	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2749	}
2750
2751	/* EPT requires a page-walk length of 4. */
2752	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2753	{
2754	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2755	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2756	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2757	}
2758	}
2759	else
2760	{
2761	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2762	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2763	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2764	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2765	}
2766	}
2767
2768	/*
2769	* Determine optimal flush type for VPID.
2770	*/
2771	if (pVM->hm.s.vmx.fVpid)
2772	{
2773	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2774	{
2775	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2776	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2777	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2778	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2779	else
2780	{
2781	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2782	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2783	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2784	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2785	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2786	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2787	pVM->hm.s.vmx.fVpid = false;
2788	}
2789	}
2790	else
2791	{
2792	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2793	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2794	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2795	pVM->hm.s.vmx.fVpid = false;
2796	}
2797	}
2798
2799	/*
2800	* Setup the handler for flushing tagged-TLBs.
2801	*/
2802	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2803	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2804	else if (pVM->hm.s.fNestedPaging)
2805	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2806	else if (pVM->hm.s.vmx.fVpid)
2807	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2808	else
2809	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2810	return VINF_SUCCESS;
2811	}
2812
2813
2814	/**
2815	* Sets up the virtual-APIC page address for the VMCS.
2816	*
2817	* @returns VBox status code.
2818	* @param pVCpu The cross context virtual CPU structure.
2819	* @param pVmcsInfo The VMCS info. object.
2820	*/
2821	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2822	{
2823	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2824	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
2825	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
2826	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2827	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
2828	}
2829
2830
2831	/**
2832	* Sets up the MSR-bitmap address for the VMCS.
2833	*
2834	* @returns VBox status code.
2835	* @param pVCpu The cross context virtual CPU structure.
2836	* @param pVmcsInfo The VMCS info. object.
2837	*/
2838	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2839	{
2840	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2841	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
2842	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
2843	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2844	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
2845	}
2846
2847
2848	/**
2849	* Sets up the APIC-access page address for the VMCS.
2850	*
2851	* @returns VBox status code.
2852	* @param pVCpu The cross context virtual CPU structure.
2853	*/
2854	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
2855	{
2856	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
2857	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
2858	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2859	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
2860	}
2861
2862
2863	/**
2864	* Sets up the VMCS link pointer for the VMCS.
2865	*
2866	* @returns VBox status code.
2867	* @param pVCpu The cross context virtual CPU structure.
2868	* @param pVmcsInfo The VMCS info. object.
2869	*/
2870	DECLINLINE(int) hmR0VmxSetupVmcsLinkPtr(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2871	{
2872	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2873	uint64_t const u64VmcsLinkPtr = pVmcsInfo->u64VmcsLinkPtr;
2874	Assert(u64VmcsLinkPtr == UINT64_C(0xffffffffffffffff)); /* Bits 63:0 MB1. */
2875	return VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, u64VmcsLinkPtr);
2876	}
2877
2878
2879	/**
2880	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
2881	* in the VMCS.
2882	*
2883	* @returns VBox status code.
2884	* @param pVCpu The cross context virtual CPU structure.
2885	* @param pVmcsInfo The VMCS info. object.
2886	*/
2887	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2888	{
2889	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2890
2891	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
2892	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
2893	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2894
2895	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
2896	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
2897	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
2898
2899	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
2900	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
2901	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2902
2903	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
2904	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
2905	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
2906	AssertRCReturn(rc, rc);
2907	return VINF_SUCCESS;
2908	}
2909
2910
2911	/**
2912	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
2913	*
2914	* @param pVCpu The cross context virtual CPU structure.
2915	* @param pVmcsInfo The VMCS info. object.
2916	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2917	*/
2918	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2919	{
2920	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
2921
2922	/*
2923	* The guest can access the following MSRs (read, write) without causing
2924	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
2925	*/
2926	PVM pVM = pVCpu->CTX_SUFF(pVM);
2927	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
2928	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
2929	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
2930	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2931	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
2932
2933	/*
2934	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
2935	* associated with then. We never need to intercept access (writes need to be
2936	* executed without causing a VM-exit, reads will #GP fault anyway).
2937	*
2938	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
2939	* read/write them. We swap the the guest/host MSR value using the
2940	* auto-load/store MSR area.
2941	*/
2942	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2943	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
2944	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
2945	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
2946	if (pVM->cpum.ro.GuestFeatures.fIbrs)
2947	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
2948
2949	#if HC_ARCH_BITS == 64
2950	/*
2951	* Allow full read/write access for the following MSRs (mandatory for VT-x)
2952	* required for 64-bit guests.
2953	*/
2954	if (pVM->hm.s.fAllow64BitGuests)
2955	{
2956	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
2957	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
2958	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
2959	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2960	}
2961	#endif
2962
2963	/*
2964	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
2965	*/
2966	#ifdef VBOX_STRICT
2967	Assert(pVmcsInfo->pvMsrBitmap);
2968	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
2969	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
2970	#endif
2971	}
2972
2973
2974	/**
2975	* Sets up pin-based VM-execution controls in the VMCS.
2976	*
2977	* @returns VBox status code.
2978	* @param pVCpu The cross context virtual CPU structure.
2979	* @param pVmcsInfo The VMCS info. object.
2980	*/
2981	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2982	{
2983	PVM pVM = pVCpu->CTX_SUFF(pVM);
2984	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
2985	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2986
2987	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2988	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2989
2990	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2991	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2992
2993	/* Enable the VMX-preemption timer. */
2994	if (pVM->hm.s.vmx.fUsePreemptTimer)
2995	{
2996	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2997	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
2998	}
2999
3000	#if 0
3001	/* Enable posted-interrupt processing. */
3002	if (pVM->hm.s.fPostedIntrs)
3003	{
3004	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
3005	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
3006	fVal \|= VMX_PIN_CTLS_POSTED_INT;
3007	}
3008	#endif
3009
3010	if ((fVal & fZap) != fVal)
3011	{
3012	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3013	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
3014	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
3015	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3016	}
3017
3018	/* Commit it to the VMCS and update our cache. */
3019	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
3020	AssertRCReturn(rc, rc);
3021	pVmcsInfo->u32PinCtls = fVal;
3022
3023	return VINF_SUCCESS;
3024	}
3025
3026
3027	/**
3028	* Sets up secondary processor-based VM-execution controls in the VMCS.
3029	*
3030	* @returns VBox status code.
3031	* @param pVCpu The cross context virtual CPU structure.
3032	* @param pVmcsInfo The VMCS info. object.
3033	*/
3034	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3035	{
3036	PVM pVM = pVCpu->CTX_SUFF(pVM);
3037	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
3038	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3039
3040	/* WBINVD causes a VM-exit. */
3041	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
3042	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
3043
3044	/* Enable EPT (aka nested-paging). */
3045	if (pVM->hm.s.fNestedPaging)
3046	fVal \|= VMX_PROC_CTLS2_EPT;
3047
3048	/* Enable the INVPCID instruction if supported by the hardware and we expose
3049	it to the guest. Without this, guest executing INVPCID would cause a #UD. */
3050	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
3051	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
3052	fVal \|= VMX_PROC_CTLS2_INVPCID;
3053
3054	/* Enable VPID. */
3055	if (pVM->hm.s.vmx.fVpid)
3056	fVal \|= VMX_PROC_CTLS2_VPID;
3057
3058	/* Enable unrestricted guest execution. */
3059	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3060	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3061
3062	#if 0
3063	if (pVM->hm.s.fVirtApicRegs)
3064	{
3065	/* Enable APIC-register virtualization. */
3066	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3067	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3068
3069	/* Enable virtual-interrupt delivery. */
3070	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3071	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3072	}
3073	#endif
3074
3075	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
3076	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3077	* done dynamically. */
3078	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3079	{
3080	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3081	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3082	AssertRCReturn(rc, rc);
3083	}
3084
3085	/* Enable the RDTSCP instruction if supported by the hardware and we expose
3086	it to the guest. Without this, guest executing RDTSCP would cause a #UD. */
3087	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
3088	&& pVM->cpum.ro.GuestFeatures.fRdTscP)
3089	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3090
3091	/* Enable Pause-Loop exiting. */
3092	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
3093	&& pVM->hm.s.vmx.cPleGapTicks
3094	&& pVM->hm.s.vmx.cPleWindowTicks)
3095	{
3096	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3097
3098	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3099	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3100	AssertRCReturn(rc, rc);
3101	}
3102
3103	if ((fVal & fZap) != fVal)
3104	{
3105	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3106	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3107	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3108	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3109	}
3110
3111	/* Commit it to the VMCS and update our cache. */
3112	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3113	AssertRCReturn(rc, rc);
3114	pVmcsInfo->u32ProcCtls2 = fVal;
3115
3116	return VINF_SUCCESS;
3117	}
3118
3119
3120	/**
3121	* Sets up processor-based VM-execution controls in the VMCS.
3122	*
3123	* @returns VBox status code.
3124	* @param pVCpu The cross context virtual CPU structure.
3125	* @param pVmcsInfo The VMCS info. object.
3126	*/
3127	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3128	{
3129	PVM pVM = pVCpu->CTX_SUFF(pVM);
3130
3131	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3132	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3133
3134	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3135	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3136	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3137	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3138	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3139	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3140	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3141
3142	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3143	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3144	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3145	{
3146	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3147	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3148	}
3149
3150	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3151	if (!pVM->hm.s.fNestedPaging)
3152	{
3153	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3154	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3155	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3156	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3157	}
3158
3159	/* Use TPR shadowing if supported by the CPU. */
3160	if ( PDMHasApic(pVM)
3161	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3162	{
3163	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3164	/* CR8 writes cause a VM-exit based on TPR threshold. */
3165	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3166	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3167	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3168	AssertRCReturn(rc, rc);
3169	}
3170	else
3171	{
3172	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3173	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3174	if (pVM->hm.s.fAllow64BitGuests)
3175	{
3176	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3177	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3178	}
3179	}
3180
3181	/* Use MSR-bitmaps if supported by the CPU. */
3182	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3183	{
3184	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3185	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3186	AssertRCReturn(rc, rc);
3187	}
3188
3189	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3190	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3191	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3192
3193	if ((fVal & fZap) != fVal)
3194	{
3195	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3196	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3197	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3198	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3199	}
3200
3201	/* Commit it to the VMCS and update our cache. */
3202	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3203	AssertRCReturn(rc, rc);
3204	pVmcsInfo->u32ProcCtls = fVal;
3205
3206	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3207	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3208	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo, false /* fIsNstGstVmcs */);
3209
3210	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3211	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3212	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3213
3214	/* Sanity check, should not really happen. */
3215	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3216	{ /* likely */ }
3217	else
3218	{
3219	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3220	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3221	}
3222
3223	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3224	return VINF_SUCCESS;
3225	}
3226
3227
3228	/**
3229	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3230	* Processor-based VM-execution) control fields in the VMCS.
3231	*
3232	* @returns VBox status code.
3233	* @param pVCpu The cross context virtual CPU structure.
3234	* @param pVmcsInfo The VMCS info. object.
3235	*/
3236	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3237	{
3238	/* Set the auto-load/store MSR area addresses in the VMCS. */
3239	int rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3240	if (RT_SUCCESS(rc))
3241	{
3242	/* Set the VMCS link pointer in the VMCS. */
3243	rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3244	if (RT_SUCCESS(rc))
3245	{
3246	/* Set the CR0/CR4 guest/host mask. */
3247	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3248	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3249	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3250	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3251	if (RT_SUCCESS(rc))
3252	{
3253	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3254	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3255	return VINF_SUCCESS;
3256	}
3257	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3258	}
3259	else
3260	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3261	}
3262	else
3263	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3264	return rc;
3265	}
3266
3267
3268	/**
3269	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3270	*
3271	* We shall setup those exception intercepts that don't change during the
3272	* lifetime of the VM here. The rest are done dynamically while loading the
3273	* guest state.
3274	*
3275	* @returns VBox status code.
3276	* @param pVCpu The cross context virtual CPU structure.
3277	* @param pVmcsInfo The VMCS info. object.
3278	*/
3279	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3280	{
3281	/*
3282	* The following exceptions are always intercepted:
3283	*
3284	* #AC - To prevent the guest from hanging the CPU.
3285	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3286	* recursive #DBs can cause a CPU hang.
3287	* #PF - To sync our shadow page tables when nested-paging is not used.
3288	*/
3289	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3290	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3291	\| RT_BIT(X86_XCPT_DB)
3292	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3293
3294	/* Commit it to the VMCS. */
3295	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3296	AssertRCReturn(rc, rc);
3297
3298	/* Update our cache of the exception bitmap. */
3299	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3300	return VINF_SUCCESS;
3301	}
3302
3303
3304	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3305	/**
3306	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3307	*
3308	* @returns VBox status code.
3309	* @param pVCpu The cross context virtual CPU structure.
3310	* @param pVmcsInfo The VMCS info. object.
3311	*/
3312	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3313	{
3314	PVM pVM = pVCpu->CTX_SUFF(pVM);
3315	int rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3316	if (RT_SUCCESS(rc))
3317	{
3318	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3319	if (RT_SUCCESS(rc))
3320	{
3321	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3322	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3323	if (RT_SUCCESS(rc))
3324	{
3325	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3326	rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3327	if (RT_SUCCESS(rc))
3328	return VINF_SUCCESS;
3329
3330	LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3331	}
3332	else
3333	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3334	}
3335	else
3336	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3337	}
3338	else
3339	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3340
3341	return rc;
3342	}
3343	#endif
3344
3345
3346	/**
3347	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3348	* VMX.
3349	*
3350	* @returns VBox status code.
3351	* @param pVCpu The cross context virtual CPU structure.
3352	* @param pVmcsInfo The VMCS info. object.
3353	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3354	*/
3355	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3356	{
3357	Assert(pVmcsInfo);
3358	Assert(pVmcsInfo->pvVmcs);
3359	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3360
3361	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3362	PVM pVM = pVCpu->CTX_SUFF(pVM);
3363	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3364	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3365
3366	LogFlowFunc(("\n"));
3367
3368	/*
3369	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3370	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3371	*/
3372	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3373	if (RT_SUCCESS(rc))
3374	{
3375	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3376	if (RT_SUCCESS(rc))
3377	{
3378	if (!fIsNstGstVmcs)
3379	{
3380	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3381	if (RT_SUCCESS(rc))
3382	{
3383	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
3384	if (RT_SUCCESS(rc))
3385	{
3386	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
3387	if (RT_SUCCESS(rc))
3388	{
3389	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
3390	if (RT_SUCCESS(rc))
3391	{ /* likely */ }
3392	else
3393	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
3394	}
3395	else
3396	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
3397	}
3398	else
3399	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
3400	}
3401	else
3402	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
3403	}
3404	else
3405	{
3406	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3407	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
3408	if (RT_SUCCESS(rc))
3409	{ /* likely */ }
3410	else
3411	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
3412	#else
3413	AssertFailed();
3414	#endif
3415	}
3416	}
3417	else
3418	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
3419	}
3420	else
3421	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
3422
3423	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
3424	if (RT_SUCCESS(rc))
3425	{
3426	rc = hmR0VmxClearVmcs(pVmcsInfo);
3427	if (RT_SUCCESS(rc))
3428	{ /* likely */ }
3429	else
3430	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
3431	}
3432
3433	/*
3434	* Update the last-error record both for failures and success, so we
3435	* can propagate the status code back to ring-3 for diagnostics.
3436	*/
3437	hmR0VmxUpdateErrorRecord(pVCpu, rc);
3438	NOREF(pszVmcs);
3439	return rc;
3440	}
3441
3442
3443	/**
3444	* Does global VT-x initialization (called during module initialization).
3445	*
3446	* @returns VBox status code.
3447	*/
3448	VMMR0DECL(int) VMXR0GlobalInit(void)
3449	{
3450	#ifdef HMVMX_USE_FUNCTION_TABLE
3451	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
3452	# ifdef VBOX_STRICT
3453	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
3454	Assert(g_apfnVMExitHandlers[i]);
3455	# endif
3456	#endif
3457	return VINF_SUCCESS;
3458	}
3459
3460
3461	/**
3462	* Does global VT-x termination (called during module termination).
3463	*/
3464	VMMR0DECL(void) VMXR0GlobalTerm()
3465	{
3466	/* Nothing to do currently. */
3467	}
3468
3469
3470	/**
3471	* Sets up and activates VT-x on the current CPU.
3472	*
3473	* @returns VBox status code.
3474	* @param pHostCpu The HM physical-CPU structure.
3475	* @param pVM The cross context VM structure. Can be
3476	* NULL after a host resume operation.
3477	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
3478	* fEnabledByHost is @c true).
3479	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
3480	* @a fEnabledByHost is @c true).
3481	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
3482	* enable VT-x on the host.
3483	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
3484	*/
3485	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
3486	PCSUPHWVIRTMSRS pHwvirtMsrs)
3487	{
3488	Assert(pHostCpu);
3489	Assert(pHwvirtMsrs);
3490	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3491
3492	/* Enable VT-x if it's not already enabled by the host. */
3493	if (!fEnabledByHost)
3494	{
3495	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
3496	if (RT_FAILURE(rc))
3497	return rc;
3498	}
3499
3500	/*
3501	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
3502	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
3503	* invalidated when flushing by VPID.
3504	*/
3505	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3506	{
3507	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
3508	pHostCpu->fFlushAsidBeforeUse = false;
3509	}
3510	else
3511	pHostCpu->fFlushAsidBeforeUse = true;
3512
3513	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
3514	++pHostCpu->cTlbFlushes;
3515
3516	return VINF_SUCCESS;
3517	}
3518
3519
3520	/**
3521	* Deactivates VT-x on the current CPU.
3522	*
3523	* @returns VBox status code.
3524	* @param pvCpuPage Pointer to the VMXON region.
3525	* @param HCPhysCpuPage Physical address of the VMXON region.
3526	*
3527	* @remarks This function should never be called when SUPR0EnableVTx() or
3528	* similar was used to enable VT-x on the host.
3529	*/
3530	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
3531	{
3532	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
3533
3534	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3535	return hmR0VmxLeaveRootMode();
3536	}
3537
3538
3539	/**
3540	* Does per-VM VT-x initialization.
3541	*
3542	* @returns VBox status code.
3543	* @param pVM The cross context VM structure.
3544	*/
3545	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
3546	{
3547	LogFlowFunc(("pVM=%p\n", pVM));
3548
3549	int rc = hmR0VmxStructsAlloc(pVM);
3550	if (RT_FAILURE(rc))
3551	{
3552	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
3553	return rc;
3554	}
3555
3556	return VINF_SUCCESS;
3557	}
3558
3559
3560	/**
3561	* Does per-VM VT-x termination.
3562	*
3563	* @returns VBox status code.
3564	* @param pVM The cross context VM structure.
3565	*/
3566	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
3567	{
3568	LogFlowFunc(("pVM=%p\n", pVM));
3569
3570	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3571	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
3572	{
3573	Assert(pVM->hm.s.vmx.pvScratch);
3574	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
3575	}
3576	#endif
3577	hmR0VmxStructsFree(pVM);
3578	return VINF_SUCCESS;
3579	}
3580
3581
3582	/**
3583	* Sets up the VM for execution using hardware-assisted VMX.
3584	* This function is only called once per-VM during initialization.
3585	*
3586	* @returns VBox status code.
3587	* @param pVM The cross context VM structure.
3588	*/
3589	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
3590	{
3591	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
3592	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3593
3594	LogFlowFunc(("pVM=%p\n", pVM));
3595
3596	/*
3597	* At least verify if VMX is enabled, since we can't check if we're in
3598	* VMX root mode or not without causing a #GP.
3599	*/
3600	RTCCUINTREG const uHostCR4 = ASMGetCR4();
3601	if (RT_LIKELY(uHostCR4 & X86_CR4_VMXE))
3602	{ /* likely */ }
3603	else
3604	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
3605
3606	/*
3607	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
3608	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
3609	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
3610	*/
3611	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3612	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
3613	\|\| !pVM->hm.s.vmx.pRealModeTSS))
3614	{
3615	LogRelFunc(("Invalid real-on-v86 state.\n"));
3616	return VERR_INTERNAL_ERROR;
3617	}
3618
3619	/* Initialize these always, see hmR3InitFinalizeR0().*/
3620	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
3621	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
3622
3623	/* Setup the tagged-TLB flush handlers. */
3624	int rc = hmR0VmxSetupTaggedTlb(pVM);
3625	if (RT_FAILURE(rc))
3626	{
3627	LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
3628	return rc;
3629	}
3630
3631	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
3632	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
3633	#if HC_ARCH_BITS == 64
3634	if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
3635	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
3636	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
3637	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
3638	#endif
3639
3640	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3641	{
3642	PVMCPU pVCpu = &pVM->aCpus[idCpu];
3643	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
3644
3645	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
3646	if (RT_SUCCESS(rc))
3647	{
3648	#if HC_ARCH_BITS == 32
3649	hmR0VmxInitVmcsReadCache(pVCpu);
3650	#endif
3651	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3652	if (pVM->cpum.ro.GuestFeatures.fVmx)
3653	{
3654	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
3655	if (RT_SUCCESS(rc))
3656	{ /* likely */ }
3657	else
3658	{
3659	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
3660	return rc;
3661	}
3662	}
3663	#endif
3664	}
3665	else
3666	{
3667	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
3668	return rc;
3669	}
3670	}
3671
3672	return VINF_SUCCESS;
3673	}
3674
3675
3676	#if HC_ARCH_BITS == 32
3677	# ifdef VBOX_ENABLE_64_BITS_GUESTS
3678	/**
3679	* Check if guest state allows safe use of 32-bit switcher again.
3680	*
3681	* Segment bases and protected mode structures must be 32-bit addressable
3682	* because the 32-bit switcher will ignore high dword when writing these VMCS
3683	* fields. See @bugref{8432} for details.
3684	*
3685	* @returns true if safe, false if must continue to use the 64-bit switcher.
3686	* @param pCtx Pointer to the guest-CPU context.
3687	*
3688	* @remarks No-long-jump zone!!!
3689	*/
3690	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
3691	{
3692	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
3693	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
3694	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3695	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3696	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
3697	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3698	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
3699	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
3700	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3701	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3702
3703	/* All good, bases are 32-bit. */
3704	return true;
3705	}
3706	# endif /* VBOX_ENABLE_64_BITS_GUESTS */
3707
3708	# ifdef VBOX_STRICT
3709	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
3710	{
3711	switch (idxField)
3712	{
3713	case VMX_VMCS_GUEST_RIP:
3714	case VMX_VMCS_GUEST_RSP:
3715	case VMX_VMCS_GUEST_SYSENTER_EIP:
3716	case VMX_VMCS_GUEST_SYSENTER_ESP:
3717	case VMX_VMCS_GUEST_GDTR_BASE:
3718	case VMX_VMCS_GUEST_IDTR_BASE:
3719	case VMX_VMCS_GUEST_CS_BASE:
3720	case VMX_VMCS_GUEST_DS_BASE:
3721	case VMX_VMCS_GUEST_ES_BASE:
3722	case VMX_VMCS_GUEST_FS_BASE:
3723	case VMX_VMCS_GUEST_GS_BASE:
3724	case VMX_VMCS_GUEST_SS_BASE:
3725	case VMX_VMCS_GUEST_LDTR_BASE:
3726	case VMX_VMCS_GUEST_TR_BASE:
3727	case VMX_VMCS_GUEST_CR3:
3728	return true;
3729	}
3730	return false;
3731	}
3732
3733	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3734	{
3735	switch (idxField)
3736	{
3737	/* Read-only fields. */
3738	case VMX_VMCS_RO_EXIT_QUALIFICATION:
3739	return true;
3740	}
3741	/* Remaining readable fields should also be writable. */
3742	return hmR0VmxIsValidWriteField(idxField);
3743	}
3744	# endif /* VBOX_STRICT */
3745
3746
3747	/**
3748	* Executes the specified handler in 64-bit mode.
3749	*
3750	* @returns VBox status code (no informational status codes).
3751	* @param pVCpu The cross context virtual CPU structure.
3752	* @param enmOp The operation to perform.
3753	* @param cParams Number of parameters.
3754	* @param paParam Array of 32-bit parameters.
3755	*/
3756	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
3757	{
3758	PVM pVM = pVCpu->CTX_SUFF(pVM);
3759	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
3760	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
3761	Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
3762	Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
3763
3764	#ifdef VBOX_STRICT
3765	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
3766	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
3767
3768	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
3769	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
3770	#endif
3771
3772	/* Disable interrupts. */
3773	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
3774
3775	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
3776	RTCPUID idHostCpu = RTMpCpuId();
3777	CPUMR0SetLApic(pVCpu, idHostCpu);
3778	#endif
3779
3780	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
3781
3782	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3783	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3784
3785	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
3786	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3787	hmR0VmxClearVmcs(pVmcsInfo);
3788
3789	/* Leave VMX root mode and disable VMX. */
3790	VMXDisable();
3791	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3792
3793	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
3794	CPUMSetHyperEIP(pVCpu, enmOp);
3795	for (int i = (int)cParams - 1; i >= 0; i--)
3796	CPUMPushHyper(pVCpu, paParam[i]);
3797
3798	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
3799
3800	/* Call the switcher. */
3801	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
3802	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
3803
3804	/* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
3805	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
3806
3807	/* Re-enter VMX root mode. */
3808	int rc2 = VMXEnable(HCPhysCpuPage);
3809	if (RT_FAILURE(rc2))
3810	{
3811	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3812	ASMSetFlags(fOldEFlags);
3813	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
3814	return rc2;
3815	}
3816
3817	/* Restore the VMCS as the current VMCS. */
3818	rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
3819	AssertRC(rc2);
3820	Assert(!(ASMGetFlags() & X86_EFL_IF));
3821	ASMSetFlags(fOldEFlags);
3822	return rc;
3823	}
3824
3825
3826	/**
3827	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
3828	* supporting 64-bit guests.
3829	*
3830	* @returns VBox status code.
3831	* @param fResume Whether to VMLAUNCH or VMRESUME.
3832	* @param pCtx Pointer to the guest-CPU context.
3833	* @param pCache Pointer to the VMCS batch cache.
3834	* @param pVM The cross context VM structure.
3835	* @param pVCpu The cross context virtual CPU structure.
3836	*/
3837	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
3838	{
3839	NOREF(fResume);
3840
3841	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3842	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3843	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3844
3845	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3846	pCache->uPos = 1;
3847	pCache->interPD = PGMGetInterPaeCR3(pVM);
3848	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
3849	#endif
3850
3851	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3852	pCache->TestIn.HCPhysCpuPage = 0;
3853	pCache->TestIn.HCPhysVmcs = 0;
3854	pCache->TestIn.pCache = 0;
3855	pCache->TestOut.HCPhysVmcs = 0;
3856	pCache->TestOut.pCache = 0;
3857	pCache->TestOut.pCtx = 0;
3858	pCache->TestOut.eflags = 0;
3859	#else
3860	NOREF(pCache);
3861	#endif
3862
3863	uint32_t aParam[10];
3864	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
3865	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
3866	aParam[2] = RT_LO_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
3867	aParam[3] = RT_HI_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
3868	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
3869	aParam[5] = 0;
3870	aParam[6] = VM_RC_ADDR(pVM, pVM);
3871	aParam[7] = 0;
3872	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
3873	aParam[9] = 0;
3874
3875	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3876	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
3877	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
3878	#endif
3879	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
3880
3881	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3882	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
3883	Assert(pCtx->dr[4] == 10);
3884	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
3885	#endif
3886
3887	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3888	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
3889	AssertMsg(pCache->TestIn.HCPhysVmcs == pVmcsInfo->HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3890	pVmcsInfo->HCPhysVmcs));
3891	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3892	pCache->TestOut.HCPhysVmcs));
3893	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
3894	pCache->TestOut.pCache));
3895	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
3896	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
3897	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
3898	pCache->TestOut.pCtx));
3899	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
3900	#endif
3901	NOREF(pCtx);
3902	return rc;
3903	}
3904	#endif
3905
3906
3907	/**
3908	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
3909	* the VMCS.
3910	*
3911	* @returns VBox status code.
3912	*/
3913	static int hmR0VmxExportHostControlRegs(void)
3914	{
3915	RTCCUINTREG uReg = ASMGetCR0();
3916	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
3917	AssertRCReturn(rc, rc);
3918
3919	uReg = ASMGetCR3();
3920	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
3921	AssertRCReturn(rc, rc);
3922
3923	uReg = ASMGetCR4();
3924	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
3925	AssertRCReturn(rc, rc);
3926	return rc;
3927	}
3928
3929
3930	/**
3931	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
3932	* the host-state area in the VMCS.
3933	*
3934	* @returns VBox status code.
3935	* @param pVCpu The cross context virtual CPU structure.
3936	*/
3937	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
3938	{
3939	#if HC_ARCH_BITS == 64
3940	/**
3941	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
3942	* requirements. See hmR0VmxExportHostSegmentRegs().
3943	*/
3944	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
3945	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
3946	{ \
3947	bool fValidSelector = true; \
3948	if ((selValue) & X86_SEL_LDT) \
3949	{ \
3950	uint32_t uAttr = ASMGetSegAttr((selValue)); \
3951	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
3952	} \
3953	if (fValidSelector) \
3954	{ \
3955	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
3956	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
3957	} \
3958	(selValue) = 0; \
3959	}
3960
3961	/*
3962	* If we've executed guest code using hardware-assisted VMX, the host-state bits
3963	* will be messed up. We should -not- save the messed up state without restoring
3964	* the original host-state, see @bugref{7240}.
3965	*
3966	* This apparently can happen (most likely the FPU changes), deal with it rather than
3967	* asserting. Was observed booting Solaris 10u10 32-bit guest.
3968	*/
3969	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
3970	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
3971	{
3972	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
3973	pVCpu->idCpu));
3974	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
3975	}
3976	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
3977	#else
3978	RT_NOREF(pVCpu);
3979	#endif
3980
3981	/*
3982	* Host DS, ES, FS and GS segment registers.
3983	*/
3984	#if HC_ARCH_BITS == 64
3985	RTSEL uSelDS = ASMGetDS();
3986	RTSEL uSelES = ASMGetES();
3987	RTSEL uSelFS = ASMGetFS();
3988	RTSEL uSelGS = ASMGetGS();
3989	#else
3990	RTSEL uSelDS = 0;
3991	RTSEL uSelES = 0;
3992	RTSEL uSelFS = 0;
3993	RTSEL uSelGS = 0;
3994	#endif
3995
3996	/*
3997	* Host CS and SS segment registers.
3998	*/
3999	RTSEL uSelCS = ASMGetCS();
4000	RTSEL uSelSS = ASMGetSS();
4001
4002	/*
4003	* Host TR segment register.
4004	*/
4005	RTSEL uSelTR = ASMGetTR();
4006
4007	#if HC_ARCH_BITS == 64
4008	/*
4009	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
4010	* gain VM-entry and restore them before we get preempted.
4011	*
4012	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
4013	*/
4014	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
4015	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
4016	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
4017	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
4018	# undef VMXLOCAL_ADJUST_HOST_SEG
4019	#endif
4020
4021	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
4022	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
4023	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
4024	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
4025	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
4026	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
4027	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
4028	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
4029	Assert(uSelCS);
4030	Assert(uSelTR);
4031
4032	/* Write these host selector fields into the host-state area in the VMCS. */
4033	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
4034	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
4035	#if HC_ARCH_BITS == 64
4036	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
4037	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
4038	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
4039	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
4040	#else
4041	NOREF(uSelDS);
4042	NOREF(uSelES);
4043	NOREF(uSelFS);
4044	NOREF(uSelGS);
4045	#endif
4046	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
4047	AssertRCReturn(rc, rc);
4048
4049	/*
4050	* Host GDTR and IDTR.
4051	*/
4052	RTGDTR Gdtr;
4053	RTIDTR Idtr;
4054	RT_ZERO(Gdtr);
4055	RT_ZERO(Idtr);
4056	ASMGetGDTR(&Gdtr);
4057	ASMGetIDTR(&Idtr);
4058	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
4059	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
4060	AssertRCReturn(rc, rc);
4061
4062	#if HC_ARCH_BITS == 64
4063	/*
4064	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4065	* them to the maximum limit (0xffff) on every VM-exit.
4066	*/
4067	if (Gdtr.cbGdt != 0xffff)
4068	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4069
4070	/*
4071	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4072	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4073	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4074	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4075	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4076	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4077	* at 0xffff on hosts where we are sure it won't cause trouble.
4078	*/
4079	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4080	if (Idtr.cbIdt < 0x0fff)
4081	# else
4082	if (Idtr.cbIdt != 0xffff)
4083	# endif
4084	{
4085	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4086	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4087	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4088	}
4089	#endif
4090
4091	/*
4092	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4093	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4094	* RPL should be too in most cases.
4095	*/
4096	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4097	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4098
4099	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4100	#if HC_ARCH_BITS == 64
4101	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4102
4103	/*
4104	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4105	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4106	* restoration if the host has something else. Task switching is not supported in 64-bit
4107	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4108	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4109	*
4110	* [1] See Intel spec. 3.5 "System Descriptor Types".
4111	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4112	*/
4113	PVM pVM = pVCpu->CTX_SUFF(pVM);
4114	Assert(pDesc->System.u4Type == 11);
4115	if ( pDesc->System.u16LimitLow != 0x67
4116	\|\| pDesc->System.u4LimitHigh)
4117	{
4118	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4119	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4120	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4121	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4122	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4123	}
4124
4125	/*
4126	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4127	*/
4128	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4129	{
4130	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4131	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4132	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4133	{
4134	/* The GDT is read-only but the writable GDT is available. */
4135	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4136	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4137	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4138	AssertRCReturn(rc, rc);
4139	}
4140	}
4141	#else
4142	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4143	#endif
4144	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4145	AssertRCReturn(rc, rc);
4146
4147	/*
4148	* Host FS base and GS base.
4149	*/
4150	#if HC_ARCH_BITS == 64
4151	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4152	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4153	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4154	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4155	AssertRCReturn(rc, rc);
4156
4157	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4158	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4159	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4160	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4161	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4162	#endif
4163	return VINF_SUCCESS;
4164	}
4165
4166
4167	/**
4168	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4169	* host-state area of the VMCS.
4170	*
4171	* These MSRs will be automatically restored on the host after every successful
4172	* VM-exit.
4173	*
4174	* @returns VBox status code.
4175	* @param pVCpu The cross context virtual CPU structure.
4176	*
4177	* @remarks No-long-jump zone!!!
4178	*/
4179	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4180	{
4181	AssertPtr(pVCpu);
4182
4183	/*
4184	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4185	* rather than swapping them on every VM-entry.
4186	*/
4187	hmR0VmxLazySaveHostMsrs(pVCpu);
4188
4189	/*
4190	* Host Sysenter MSRs.
4191	*/
4192	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4193	#if HC_ARCH_BITS == 32
4194	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4195	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4196	#else
4197	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4198	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4199	#endif
4200	AssertRCReturn(rc, rc);
4201
4202	/*
4203	* Host EFER MSR.
4204	*
4205	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4206	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4207	*/
4208	PVM pVM = pVCpu->CTX_SUFF(pVM);
4209	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4210	{
4211	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4212	AssertRCReturn(rc, rc);
4213	}
4214
4215	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4216	* hmR0VmxExportGuestEntryExitCtls(). */
4217
4218	return VINF_SUCCESS;
4219	}
4220
4221
4222	/**
4223	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4224	*
4225	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4226	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4227	*
4228	* @returns true if we need to load guest EFER, false otherwise.
4229	* @param pVCpu The cross context virtual CPU structure.
4230	*
4231	* @remarks Requires EFER, CR4.
4232	* @remarks No-long-jump zone!!!
4233	*/
4234	static bool hmR0VmxShouldSwapEferMsr(PCVMCPU pVCpu)
4235	{
4236	#ifdef HMVMX_ALWAYS_SWAP_EFER
4237	RT_NOREF(pVCpu);
4238	return true;
4239	#else
4240	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4241	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4242	/* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4243	if (CPUMIsGuestInLongModeEx(pCtx))
4244	return false;
4245	#endif
4246
4247	PVM pVM = pVCpu->CTX_SUFF(pVM);
4248	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4249	uint64_t const u64GuestEfer = pCtx->msrEFER;
4250
4251	/*
4252	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4253	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4254	*/
4255	if ( CPUMIsGuestInLongModeEx(pCtx)
4256	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4257	return true;
4258
4259	/*
4260	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4261	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4262	*
4263	* See Intel spec. 4.5 "IA-32e Paging".
4264	* See Intel spec. 4.1.1 "Three Paging Modes".
4265	*
4266	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4267	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4268	*/
4269	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4270	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4271	if ( (pCtx->cr4 & X86_CR4_PAE)
4272	&& (pCtx->cr0 & X86_CR0_PG)
4273	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4274	{
4275	/* Assert that host is NX capable. */
4276	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4277	return true;
4278	}
4279
4280	return false;
4281	#endif
4282	}
4283
4284	/**
4285	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4286	* VMCS.
4287	*
4288	* This is typically required when the guest changes paging mode.
4289	*
4290	* @returns VBox status code.
4291	* @param pVCpu The cross context virtual CPU structure.
4292	* @param pVmxTransient The VMX-transient structure.
4293	*
4294	* @remarks Requires EFER.
4295	* @remarks No-long-jump zone!!!
4296	*/
4297	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4298	{
4299	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4300	{
4301	PVM pVM = pVCpu->CTX_SUFF(pVM);
4302	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4303
4304	/*
4305	* VM-entry controls.
4306	*/
4307	{
4308	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4309	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4310
4311	/*
4312	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4313	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4314	*
4315	* For nested-guests, this is a mandatory VM-entry control. It's also
4316	* required because we do not want to leak host bits to the nested-guest.
4317	*/
4318	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4319
4320	/*
4321	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4322	*
4323	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4324	* required to get the nested-guest working with hardware-assisted VMX execution.
4325	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested-hypervisor
4326	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4327	* here rather than while merging the guest VMCS controls.
4328	*/
4329	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4330	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4331	else
4332	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4333
4334	/*
4335	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4336	*
4337	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4338	* regardless of whether the nested-guest VMCS specifies it because we are free to
4339	* load whatever MSRs we require and we do not need to modify the guest visible copy
4340	* of the VM-entry MSR load area.
4341	*/
4342	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4343	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4344	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4345	else
4346	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4347
4348	/*
4349	* The following should -not- be set (since we're not in SMM mode):
4350	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4351	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4352	*/
4353
4354	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4355	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4356
4357	if ((fVal & fZap) == fVal)
4358	{ /* likely */ }
4359	else
4360	{
4361	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4362	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4363	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4364	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4365	}
4366
4367	/* Commit it to the VMCS. */
4368	if (pVmcsInfo->u32EntryCtls != fVal)
4369	{
4370	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4371	AssertRCReturn(rc, rc);
4372	pVmcsInfo->u32EntryCtls = fVal;
4373	}
4374	}
4375
4376	/*
4377	* VM-exit controls.
4378	*/
4379	{
4380	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4381	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4382
4383	/*
4384	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4385	* supported the 1-setting of this bit.
4386	*
4387	* For nested-guests, we set the "save debug controls" as the converse
4388	* "load debug controls" is mandatory for nested-guests anyway.
4389	*/
4390	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4391
4392	/*
4393	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4394	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4395	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4396	* hmR0VmxExportHostMsrs().
4397	*
4398	* For nested-guests, we always set this bit as we do not support 32-bit
4399	* hosts.
4400	*/
4401	#if HC_ARCH_BITS == 64
4402	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4403	#else
4404	Assert(!pVmxTransient->fIsNestedGuest);
4405	Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
4406	\|\| pVmcsInfo->pfnStartVM == VMXR0StartVM32);
4407	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
4408	if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
4409	{
4410	/* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
4411	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4412	}
4413	else
4414	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
4415	#endif
4416
4417	/*
4418	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4419	*
4420	* For nested-guests, we should use the "save IA32_EFER" control if we also
4421	* used the "load IA32_EFER" control while exporting VM-entry controls.
4422	*/
4423	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4424	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4425	{
4426	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4427	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4428	}
4429
4430	/*
4431	* Enable saving of the VMX-preemption timer value on VM-exit.
4432	* For nested-guests, currently not exposed/used.
4433	*/
4434	if ( pVM->hm.s.vmx.fUsePreemptTimer
4435	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4436	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4437
4438	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4439	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4440
4441	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4442	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4443	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4444
4445	if ((fVal & fZap) == fVal)
4446	{ /* likely */ }
4447	else
4448	{
4449	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4450	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4451	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4452	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4453	}
4454
4455	/* Commit it to the VMCS. */
4456	if (pVmcsInfo->u32ExitCtls != fVal)
4457	{
4458	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4459	AssertRCReturn(rc, rc);
4460	pVmcsInfo->u32ExitCtls = fVal;
4461	}
4462	}
4463
4464	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4465	}
4466	return VINF_SUCCESS;
4467	}
4468
4469
4470	/**
4471	* Sets the TPR threshold in the VMCS.
4472	*
4473	* @returns VBox status code.
4474	* @param pVCpu The cross context virtual CPU structure.
4475	* @param pVmcsInfo The VMCS info. object.
4476	* @param u32TprThreshold The TPR threshold (task-priority class only).
4477	*/
4478	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4479	{
4480	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4481	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4482	RT_NOREF2(pVCpu, pVmcsInfo);
4483	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4484	}
4485
4486
4487	/**
4488	* Exports the guest APIC TPR state into the VMCS.
4489	*
4490	* @returns VBox status code.
4491	* @param pVCpu The cross context virtual CPU structure.
4492	* @param pVmxTransient The VMX-transient structure.
4493	*
4494	* @remarks No-long-jump zone!!!
4495	*/
4496	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4497	{
4498	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4499	{
4500	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4501
4502	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4503	if (!pVmxTransient->fIsNestedGuest)
4504	{
4505	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4506	&& APICIsEnabled(pVCpu))
4507	{
4508	/*
4509	* Setup TPR shadowing.
4510	*/
4511	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4512	{
4513	bool fPendingIntr = false;
4514	uint8_t u8Tpr = 0;
4515	uint8_t u8PendingIntr = 0;
4516	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4517	AssertRCReturn(rc, rc);
4518
4519	/*
4520	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4521	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4522	* priority of the pending interrupt so we can deliver the interrupt. If there
4523	* are no interrupts pending, set threshold to 0 to not cause any
4524	* TPR-below-threshold VM-exits.
4525	*/
4526	Assert(pVmcsInfo->pbVirtApic);
4527	pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
4528	uint32_t u32TprThreshold = 0;
4529	if (fPendingIntr)
4530	{
4531	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4532	(which is the Task-Priority Class). */
4533	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4534	const uint8_t u8TprPriority = u8Tpr >> 4;
4535	if (u8PendingPriority <= u8TprPriority)
4536	u32TprThreshold = u8PendingPriority;
4537	}
4538
4539	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
4540	AssertRCReturn(rc, rc);
4541	}
4542	}
4543	}
4544	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4545	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4546	}
4547	return VINF_SUCCESS;
4548	}
4549
4550
4551	/**
4552	* Gets the guest interruptibility-state.
4553	*
4554	* @returns Guest's interruptibility-state.
4555	* @param pVCpu The cross context virtual CPU structure.
4556	* @param pVmcsInfo The VMCS info. object.
4557	*
4558	* @remarks No-long-jump zone!!!
4559	*/
4560	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
4561	{
4562	/*
4563	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4564	*/
4565	uint32_t fIntrState = 0;
4566	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4567	{
4568	/* If inhibition is active, RIP and RFLAGS should've been updated
4569	(i.e. read previously from the VMCS or from ring-3). */
4570	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4571	#ifdef VBOX_STRICT
4572	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
4573	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4574	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
4575	#endif
4576	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
4577	{
4578	if (pCtx->eflags.Bits.u1IF)
4579	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4580	else
4581	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
4582	}
4583	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4584	{
4585	/*
4586	* We can clear the inhibit force flag as even if we go back to the recompiler
4587	* without executing guest code in VT-x, the flag's condition to be cleared is
4588	* met and thus the cleared state is correct.
4589	*/
4590	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4591	}
4592	}
4593
4594	/*
4595	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
4596	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
4597	* setting this would block host-NMIs and IRET will not clear the blocking.
4598	*
4599	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
4600	*
4601	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
4602	*/
4603	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
4604	&& CPUMIsGuestNmiBlocking(pVCpu))
4605	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
4606
4607	return fIntrState;
4608	}
4609
4610
4611	/**
4612	* Exports the exception intercepts required for guest execution in the VMCS.
4613	*
4614	* @returns VBox status code.
4615	* @param pVCpu The cross context virtual CPU structure.
4616	* @param pVmxTransient The VMX-transient structure.
4617	*
4618	* @remarks No-long-jump zone!!!
4619	*/
4620	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4621	{
4622	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
4623	{
4624	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
4625	if ( !pVmxTransient->fIsNestedGuest
4626	&& pVCpu->hm.s.fGIMTrapXcptUD)
4627	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
4628	else
4629	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
4630
4631	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
4632	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
4633	}
4634	return VINF_SUCCESS;
4635	}
4636
4637
4638	/**
4639	* Exports the guest's RIP into the guest-state area in the VMCS.
4640	*
4641	* @returns VBox status code.
4642	* @param pVCpu The cross context virtual CPU structure.
4643	*
4644	* @remarks No-long-jump zone!!!
4645	*/
4646	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
4647	{
4648	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4649	{
4650	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
4651
4652	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
4653	AssertRCReturn(rc, rc);
4654
4655	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
4656	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
4657	}
4658	return VINF_SUCCESS;
4659	}
4660
4661
4662	/**
4663	* Exports the guest's RSP into the guest-state area in the VMCS.
4664	*
4665	* @returns VBox status code.
4666	* @param pVCpu The cross context virtual CPU structure.
4667	*
4668	* @remarks No-long-jump zone!!!
4669	*/
4670	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
4671	{
4672	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
4673	{
4674	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
4675
4676	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
4677	AssertRCReturn(rc, rc);
4678
4679	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
4680	}
4681	return VINF_SUCCESS;
4682	}
4683
4684
4685	/**
4686	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
4687	*
4688	* @returns VBox status code.
4689	* @param pVCpu The cross context virtual CPU structure.
4690	* @param pVmxTransient The VMX-transient structure.
4691	*
4692	* @remarks No-long-jump zone!!!
4693	*/
4694	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4695	{
4696	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
4697	{
4698	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
4699
4700	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
4701	Let us assert it as such and use 32-bit VMWRITE. */
4702	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
4703	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
4704	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
4705	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
4706
4707	/*
4708	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
4709	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
4710	* can run the real-mode guest code under Virtual 8086 mode.
4711	*/
4712	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4713	if (pVmcsInfo->RealMode.fRealOnV86Active)
4714	{
4715	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4716	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4717	Assert(!pVmxTransient->fIsNestedGuest);
4718	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
4719	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
4720	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
4721	}
4722
4723	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
4724	AssertRCReturn(rc, rc);
4725
4726	/*
4727	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
4728	*
4729	* We must avoid setting any automatic debug exceptions delivery when single-stepping
4730	* through the hypervisor debugger using EFLAGS.TF.
4731	*/
4732	if ( !pVmxTransient->fIsNestedGuest
4733	&& !pVCpu->hm.s.fSingleInstruction
4734	&& fEFlags.Bits.u1TF)
4735	{
4736	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
4737	* premature trips to ring-3 esp since IEM does not yet handle it. */
4738	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
4739	AssertRCReturn(rc, rc);
4740	}
4741	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
4742	* nested-guest VMCS. */
4743
4744	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
4745	Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
4746	}
4747	return VINF_SUCCESS;
4748	}
4749
4750
4751	/**
4752	* Exports the guest CR0 control register into the guest-state area in the VMCS.
4753	*
4754	* The guest FPU state is always pre-loaded hence we don't need to bother about
4755	* sharing FPU related CR0 bits between the guest and host.
4756	*
4757	* @returns VBox status code.
4758	* @param pVCpu The cross context virtual CPU structure.
4759	* @param pVmxTransient The VMX-transient structure.
4760	*
4761	* @remarks No-long-jump zone!!!
4762	*/
4763	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4764	{
4765	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
4766	{
4767	PVM pVM = pVCpu->CTX_SUFF(pVM);
4768	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4769
4770	/*
4771	* Figure out fixed CR0 bits in VMX operation.
4772	*/
4773	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4774	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4775	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4776	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
4777	else
4778	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
4779
4780	if (!pVmxTransient->fIsNestedGuest)
4781	{
4782	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4783	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4784	uint64_t const u64ShadowCr0 = u64GuestCr0;
4785	Assert(!RT_HI_U32(u64GuestCr0));
4786
4787	/*
4788	* Setup VT-x's view of the guest CR0.
4789	*/
4790	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
4791	if (pVM->hm.s.fNestedPaging)
4792	{
4793	if (CPUMIsGuestPagingEnabled(pVCpu))
4794	{
4795	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
4796	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
4797	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
4798	}
4799	else
4800	{
4801	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
4802	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
4803	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
4804	}
4805
4806	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
4807	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4808	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
4809	}
4810	else
4811	{
4812	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
4813	u64GuestCr0 \|= X86_CR0_WP;
4814	}
4815
4816	/*
4817	* Guest FPU bits.
4818	*
4819	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
4820	* using CR0.TS.
4821	*
4822	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
4823	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
4824	*/
4825	u64GuestCr0 \|= X86_CR0_NE;
4826
4827	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
4828	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
4829
4830	/*
4831	* Update exception intercepts.
4832	*/
4833	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
4834	if (pVmcsInfo->RealMode.fRealOnV86Active)
4835	{
4836	Assert(PDMVmmDevHeapIsEnabled(pVM));
4837	Assert(pVM->hm.s.vmx.pRealModeTSS);
4838	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
4839	}
4840	else
4841	{
4842	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
4843	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
4844	if (fInterceptMF)
4845	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
4846	}
4847
4848	/* Additional intercepts for debugging, define these yourself explicitly. */
4849	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4850	uXcptBitmap \|= 0
4851	\| RT_BIT(X86_XCPT_BP)
4852	\| RT_BIT(X86_XCPT_DE)
4853	\| RT_BIT(X86_XCPT_NM)
4854	\| RT_BIT(X86_XCPT_TS)
4855	\| RT_BIT(X86_XCPT_UD)
4856	\| RT_BIT(X86_XCPT_NP)
4857	\| RT_BIT(X86_XCPT_SS)
4858	\| RT_BIT(X86_XCPT_GP)
4859	\| RT_BIT(X86_XCPT_PF)
4860	\| RT_BIT(X86_XCPT_MF)
4861	;
4862	#elif defined(HMVMX_ALWAYS_TRAP_PF)
4863	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
4864	#endif
4865	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
4866	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
4867	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
4868
4869	/* Apply the fixed CR0 bits and enable caching. */
4870	u64GuestCr0 \|= fSetCr0;
4871	u64GuestCr0 &= fZapCr0;
4872	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4873
4874	/* Commit the CR0 and related fields to the guest VMCS. */
4875	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo Fix to 64-bit when we drop 32-bit. */
4876	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4877	if (uProcCtls != pVmcsInfo->u32ProcCtls)
4878	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4879	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
4880	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
4881	AssertRCReturn(rc, rc);
4882
4883	/* Update our caches. */
4884	pVmcsInfo->u32ProcCtls = uProcCtls;
4885	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
4886
4887	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4888	}
4889	else
4890	{
4891	/*
4892	* With nested-guests, we may have extended the guest/host mask here (since we
4893	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
4894	* mask can include more bits (to read from the nested-guest CR0 read-shadow) than
4895	* the guest hypervisor originally supplied. Thus, we should, in essence, copy
4896	* those bits from the nested-guest CR0 into the nested-guest CR0 read shadow.
4897	*/
4898	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4899	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4900	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(pVCpu, &pVCpu->cpum.GstCtx);
4901	Assert(!RT_HI_U32(u64GuestCr0));
4902	Assert(u64GuestCr0 & X86_CR0_NE);
4903
4904	/* Apply the fixed CR0 bits and enable caching. */
4905	u64GuestCr0 \|= fSetCr0;
4906	u64GuestCr0 &= fZapCr0;
4907	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4908
4909	/* Commit the CR0 and CR0 read shadow to the nested-guest VMCS. */
4910	/** @todo NSTVMX: Fix to 64-bit when we drop 32-bit. */
4911	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0);
4912	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4913	AssertRCReturn(rc, rc);
4914
4915	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4916	}
4917
4918	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
4919	}
4920
4921	return VINF_SUCCESS;
4922	}
4923
4924
4925	/**
4926	* Exports the guest control registers (CR3, CR4) into the guest-state area
4927	* in the VMCS.
4928	*
4929	* @returns VBox strict status code.
4930	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
4931	* without unrestricted guest access and the VMMDev is not presently
4932	* mapped (e.g. EFI32).
4933	*
4934	* @param pVCpu The cross context virtual CPU structure.
4935	* @param pVmxTransient The VMX-transient structure.
4936	*
4937	* @remarks No-long-jump zone!!!
4938	*/
4939	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4940	{
4941	int rc = VINF_SUCCESS;
4942	PVM pVM = pVCpu->CTX_SUFF(pVM);
4943
4944	/*
4945	* Guest CR2.
4946	* It's always loaded in the assembler code. Nothing to do here.
4947	*/
4948
4949	/*
4950	* Guest CR3.
4951	*/
4952	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
4953	{
4954	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
4955
4956	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
4957	if (pVM->hm.s.fNestedPaging)
4958	{
4959	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4960	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
4961
4962	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
4963	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
4964	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
4965	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
4966
4967	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
4968	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
4969	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
4970
4971	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
4972	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
4973	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
4974	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
4975	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
4976	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
4977	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
4978
4979	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
4980	AssertRCReturn(rc, rc);
4981
4982	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4983	if ( pVM->hm.s.vmx.fUnrestrictedGuest
4984	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
4985	{
4986	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
4987	if (CPUMIsGuestInPAEModeEx(pCtx))
4988	{
4989	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
4990	AssertRCReturn(rc, rc);
4991	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
4992	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
4993	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
4994	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
4995	AssertRCReturn(rc, rc);
4996	}
4997
4998	/*
4999	* The guest's view of its CR3 is unblemished with nested paging when the
5000	* guest is using paging or we have unrestricted guest execution to handle
5001	* the guest when it's not using paging.
5002	*/
5003	GCPhysGuestCR3 = pCtx->cr3;
5004	}
5005	else
5006	{
5007	/*
5008	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
5009	* thinks it accesses physical memory directly, we use our identity-mapped
5010	* page table to map guest-linear to guest-physical addresses. EPT takes care
5011	* of translating it to host-physical addresses.
5012	*/
5013	RTGCPHYS GCPhys;
5014	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
5015
5016	/* We obtain it here every time as the guest could have relocated this PCI region. */
5017	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
5018	if (RT_SUCCESS(rc))
5019	{ /* likely */ }
5020	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
5021	{
5022	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
5023	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
5024	}
5025	else
5026	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
5027
5028	GCPhysGuestCR3 = GCPhys;
5029	}
5030
5031	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
5032	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
5033	AssertRCReturn(rc, rc);
5034	}
5035	else
5036	{
5037	/* Non-nested paging case, just use the hypervisor's CR3. */
5038	RTHCPHYS const HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
5039
5040	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
5041	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
5042	AssertRCReturn(rc, rc);
5043	}
5044
5045	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
5046	}
5047
5048	/*
5049	* Guest CR4.
5050	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
5051	*/
5052	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
5053	{
5054	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5055	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5056
5057	/*
5058	* Figure out fixed CR4 bits in VMX operation.
5059	*/
5060	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5061	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5062
5063	/*
5064	* With nested-guests, we may have extended the guest/host mask here (since we
5065	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
5066	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
5067	* the guest hypervisor originally supplied. Thus, we should, in essence, copy
5068	* those bits from the nested-guest CR4 into the nested-guest CR4 read shadow.
5069	*/
5070	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5071	uint64_t u64GuestCr4 = pCtx->cr4;
5072	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
5073	? pCtx->cr4
5074	: CPUMGetGuestVmxMaskedCr4(pVCpu, pCtx);
5075	Assert(!RT_HI_U32(u64GuestCr4));
5076
5077	/*
5078	* Setup VT-x's view of the guest CR4.
5079	*
5080	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5081	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5082	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5083	*
5084	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5085	*/
5086	if (pVmcsInfo->RealMode.fRealOnV86Active)
5087	{
5088	Assert(pVM->hm.s.vmx.pRealModeTSS);
5089	Assert(PDMVmmDevHeapIsEnabled(pVM));
5090	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5091	}
5092
5093	if (pVM->hm.s.fNestedPaging)
5094	{
5095	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5096	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5097	{
5098	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5099	u64GuestCr4 \|= X86_CR4_PSE;
5100	/* Our identity mapping is a 32-bit page directory. */
5101	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5102	}
5103	/* else use guest CR4.*/
5104	}
5105	else
5106	{
5107	Assert(!pVmxTransient->fIsNestedGuest);
5108
5109	/*
5110	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5111	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5112	*/
5113	switch (pVCpu->hm.s.enmShadowMode)
5114	{
5115	case PGMMODE_REAL: /* Real-mode. */
5116	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5117	case PGMMODE_32_BIT: /* 32-bit paging. */
5118	{
5119	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5120	break;
5121	}
5122
5123	case PGMMODE_PAE: /* PAE paging. */
5124	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5125	{
5126	u64GuestCr4 \|= X86_CR4_PAE;
5127	break;
5128	}
5129
5130	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5131	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5132	#ifdef VBOX_ENABLE_64_BITS_GUESTS
5133	break;
5134	#endif
5135	default:
5136	AssertFailed();
5137	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5138	}
5139	}
5140
5141	/* Apply the fixed CR4 bits (mainly CR4.VMXE). */
5142	u64GuestCr4 \|= fSetCr4;
5143	u64GuestCr4 &= fZapCr4;
5144
5145	/* Commit the CR4 and CR4 read shadow to the guest VMCS. */
5146	/** @todo Fix to 64-bit when we drop 32-bit. */
5147	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u64GuestCr4);
5148	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
5149	AssertRCReturn(rc, rc);
5150
5151	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5152	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5153
5154	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5155
5156	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5157	}
5158	return rc;
5159	}
5160
5161
5162	/**
5163	* Exports the guest debug registers into the guest-state area in the VMCS.
5164	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5165	*
5166	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5167	*
5168	* @returns VBox status code.
5169	* @param pVCpu The cross context virtual CPU structure.
5170	* @param pVmxTransient The VMX-transient structure.
5171	*
5172	* @remarks No-long-jump zone!!!
5173	*/
5174	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5175	{
5176	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5177
5178	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5179	* stepping. */
5180	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5181	if (pVmxTransient->fIsNestedGuest)
5182	{
5183	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5184	AssertRCReturn(rc, rc);
5185	return VINF_SUCCESS;
5186	}
5187
5188	#ifdef VBOX_STRICT
5189	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5190	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5191	{
5192	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5193	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5194	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5195	}
5196	#endif
5197
5198	bool fSteppingDB = false;
5199	bool fInterceptMovDRx = false;
5200	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5201	if (pVCpu->hm.s.fSingleInstruction)
5202	{
5203	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5204	PVM pVM = pVCpu->CTX_SUFF(pVM);
5205	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5206	{
5207	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5208	Assert(fSteppingDB == false);
5209	}
5210	else
5211	{
5212	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5213	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5214	pVCpu->hm.s.fClearTrapFlag = true;
5215	fSteppingDB = true;
5216	}
5217	}
5218
5219	uint32_t u32GuestDr7;
5220	if ( fSteppingDB
5221	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5222	{
5223	/*
5224	* Use the combined guest and host DRx values found in the hypervisor register set
5225	* because the hypervisor debugger has breakpoints active or someone is single stepping
5226	* on the host side without a monitor trap flag.
5227	*
5228	* Note! DBGF expects a clean DR6 state before executing guest code.
5229	*/
5230	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5231	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5232	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
5233	{
5234	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5235	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
5236	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
5237	}
5238	else
5239	#endif
5240	if (!CPUMIsHyperDebugStateActive(pVCpu))
5241	{
5242	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5243	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5244	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5245	}
5246
5247	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5248	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
5249	pVCpu->hm.s.fUsingHyperDR7 = true;
5250	fInterceptMovDRx = true;
5251	}
5252	else
5253	{
5254	/*
5255	* If the guest has enabled debug registers, we need to load them prior to
5256	* executing guest code so they'll trigger at the right time.
5257	*/
5258	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5259	{
5260	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5261	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5262	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
5263	{
5264	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5265	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
5266	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
5267	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5268	}
5269	else
5270	#endif
5271	if (!CPUMIsGuestDebugStateActive(pVCpu))
5272	{
5273	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5274	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5275	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5276	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5277	}
5278	Assert(!fInterceptMovDRx);
5279	}
5280	/*
5281	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5282	* must intercept #DB in order to maintain a correct DR6 guest value, and
5283	* because we need to intercept it to prevent nested #DBs from hanging the
5284	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5285	*/
5286	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5287	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
5288	&& !CPUMIsGuestDebugStateActive(pVCpu))
5289	#else
5290	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5291	#endif
5292	{
5293	fInterceptMovDRx = true;
5294	}
5295
5296	/* Update DR7 with the actual guest value. */
5297	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5298	pVCpu->hm.s.fUsingHyperDR7 = false;
5299	}
5300
5301	if (fInterceptMovDRx)
5302	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5303	else
5304	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5305
5306	/*
5307	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5308	* monitor-trap flag and update our cache.
5309	*/
5310	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5311	{
5312	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5313	AssertRCReturn(rc2, rc2);
5314	pVmcsInfo->u32ProcCtls = uProcCtls;
5315	}
5316
5317	/*
5318	* Update guest DR7.
5319	*/
5320	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
5321	AssertRCReturn(rc, rc);
5322
5323	/*
5324	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
5325	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
5326	*
5327	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5328	*/
5329	if (fSteppingDB)
5330	{
5331	Assert(pVCpu->hm.s.fSingleInstruction);
5332	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
5333
5334	uint32_t fIntrState = 0;
5335	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
5336	AssertRCReturn(rc, rc);
5337
5338	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5339	{
5340	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5341	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5342	AssertRCReturn(rc, rc);
5343	}
5344	}
5345
5346	return VINF_SUCCESS;
5347	}
5348
5349
5350	#ifdef VBOX_STRICT
5351	/**
5352	* Strict function to validate segment registers.
5353	*
5354	* @param pVCpu The cross context virtual CPU structure.
5355	* @param pVmcsInfo The VMCS info. object.
5356	*
5357	* @remarks Will import guest CR0 on strict builds during validation of
5358	* segments.
5359	*/
5360	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
5361	{
5362	/*
5363	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5364	*
5365	* The reason we check for attribute value 0 in this function and not just the unusable bit is
5366	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
5367	* unusable bit and doesn't change the guest-context value.
5368	*/
5369	PVM pVM = pVCpu->CTX_SUFF(pVM);
5370	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5371	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
5372	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
5373	&& ( !CPUMIsGuestInRealModeEx(pCtx)
5374	&& !CPUMIsGuestInV86ModeEx(pCtx)))
5375	{
5376	/* Protected mode checks */
5377	/* CS */
5378	Assert(pCtx->cs.Attr.n.u1Present);
5379	Assert(!(pCtx->cs.Attr.u & 0xf00));
5380	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
5381	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5382	\|\| !(pCtx->cs.Attr.n.u1Granularity));
5383	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
5384	\|\| (pCtx->cs.Attr.n.u1Granularity));
5385	/* CS cannot be loaded with NULL in protected mode. */
5386	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
5387	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5388	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
5389	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5390	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
5391	else
5392	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
5393	/* SS */
5394	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5395	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
5396	if ( !(pCtx->cr0 & X86_CR0_PE)
5397	\|\| pCtx->cs.Attr.n.u4Type == 3)
5398	{
5399	Assert(!pCtx->ss.Attr.n.u2Dpl);
5400	}
5401	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
5402	{
5403	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5404	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
5405	Assert(pCtx->ss.Attr.n.u1Present);
5406	Assert(!(pCtx->ss.Attr.u & 0xf00));
5407	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
5408	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
5409	\|\| !(pCtx->ss.Attr.n.u1Granularity));
5410	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
5411	\|\| (pCtx->ss.Attr.n.u1Granularity));
5412	}
5413	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
5414	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
5415	{
5416	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5417	Assert(pCtx->ds.Attr.n.u1Present);
5418	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
5419	Assert(!(pCtx->ds.Attr.u & 0xf00));
5420	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
5421	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
5422	\|\| !(pCtx->ds.Attr.n.u1Granularity));
5423	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
5424	\|\| (pCtx->ds.Attr.n.u1Granularity));
5425	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5426	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
5427	}
5428	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
5429	{
5430	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5431	Assert(pCtx->es.Attr.n.u1Present);
5432	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
5433	Assert(!(pCtx->es.Attr.u & 0xf00));
5434	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
5435	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
5436	\|\| !(pCtx->es.Attr.n.u1Granularity));
5437	Assert( !(pCtx->es.u32Limit & 0xfff00000)
5438	\|\| (pCtx->es.Attr.n.u1Granularity));
5439	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5440	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
5441	}
5442	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
5443	{
5444	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5445	Assert(pCtx->fs.Attr.n.u1Present);
5446	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
5447	Assert(!(pCtx->fs.Attr.u & 0xf00));
5448	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
5449	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
5450	\|\| !(pCtx->fs.Attr.n.u1Granularity));
5451	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
5452	\|\| (pCtx->fs.Attr.n.u1Granularity));
5453	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5454	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5455	}
5456	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
5457	{
5458	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5459	Assert(pCtx->gs.Attr.n.u1Present);
5460	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
5461	Assert(!(pCtx->gs.Attr.u & 0xf00));
5462	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
5463	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
5464	\|\| !(pCtx->gs.Attr.n.u1Granularity));
5465	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
5466	\|\| (pCtx->gs.Attr.n.u1Granularity));
5467	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5468	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5469	}
5470	/* 64-bit capable CPUs. */
5471	# if HC_ARCH_BITS == 64
5472	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5473	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
5474	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
5475	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
5476	# endif
5477	}
5478	else if ( CPUMIsGuestInV86ModeEx(pCtx)
5479	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
5480	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
5481	{
5482	/* Real and v86 mode checks. */
5483	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
5484	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
5485	if (pVmcsInfo->RealMode.fRealOnV86Active)
5486	{
5487	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
5488	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
5489	}
5490	else
5491	{
5492	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
5493	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
5494	}
5495
5496	/* CS */
5497	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
5498	Assert(pCtx->cs.u32Limit == 0xffff);
5499	Assert(u32CSAttr == 0xf3);
5500	/* SS */
5501	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
5502	Assert(pCtx->ss.u32Limit == 0xffff);
5503	Assert(u32SSAttr == 0xf3);
5504	/* DS */
5505	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
5506	Assert(pCtx->ds.u32Limit == 0xffff);
5507	Assert(u32DSAttr == 0xf3);
5508	/* ES */
5509	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
5510	Assert(pCtx->es.u32Limit == 0xffff);
5511	Assert(u32ESAttr == 0xf3);
5512	/* FS */
5513	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
5514	Assert(pCtx->fs.u32Limit == 0xffff);
5515	Assert(u32FSAttr == 0xf3);
5516	/* GS */
5517	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
5518	Assert(pCtx->gs.u32Limit == 0xffff);
5519	Assert(u32GSAttr == 0xf3);
5520	/* 64-bit capable CPUs. */
5521	# if HC_ARCH_BITS == 64
5522	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5523	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
5524	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
5525	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
5526	# endif
5527	}
5528	}
5529	#endif /* VBOX_STRICT */
5530
5531
5532	/**
5533	* Exports a guest segment register into the guest-state area in the VMCS.
5534	*
5535	* @returns VBox status code.
5536	* @param pVCpu The cross context virtual CPU structure.
5537	* @param pVmcsInfo The VMCS info. object.
5538	* @param iSegReg The segment register number (X86_SREG_XXX).
5539	* @param pSelReg Pointer to the segment selector.
5540	*
5541	* @remarks No-long-jump zone!!!
5542	*/
5543	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
5544	{
5545	Assert(iSegReg < X86_SREG_COUNT);
5546	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
5547	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
5548	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
5549	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
5550
5551	uint32_t u32Access = pSelReg->Attr.u;
5552	if (pVmcsInfo->RealMode.fRealOnV86Active)
5553	{
5554	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
5555	u32Access = 0xf3;
5556	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5557	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5558	RT_NOREF_PV(pVCpu);
5559	}
5560	else
5561	{
5562	/*
5563	* The way to differentiate between whether this is really a null selector or was just
5564	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
5565	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
5566	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
5567	* NULL selectors loaded in protected-mode have their attribute as 0.
5568	*/
5569	if (!u32Access)
5570	u32Access = X86DESCATTR_UNUSABLE;
5571	}
5572
5573	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
5574	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
5575	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
5576
5577	/*
5578	* Commit it to the VMCS.
5579	*/
5580	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
5581	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
5582	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
5583	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
5584	AssertRCReturn(rc, rc);
5585	return rc;
5586	}
5587
5588
5589	/**
5590	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
5591	* area in the VMCS.
5592	*
5593	* @returns VBox status code.
5594	* @param pVCpu The cross context virtual CPU structure.
5595	* @param pVmxTransient The VMX-transient structure.
5596	*
5597	* @remarks Will import guest CR0 on strict builds during validation of
5598	* segments.
5599	* @remarks No-long-jump zone!!!
5600	*/
5601	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5602	{
5603	int rc = VERR_INTERNAL_ERROR_5;
5604	PVM pVM = pVCpu->CTX_SUFF(pVM);
5605	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5606	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5607
5608	/*
5609	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
5610	*/
5611	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
5612	{
5613	#ifdef VBOX_WITH_REM
5614	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
5615	{
5616	Assert(!pVmxTransient->fIsNestedGuest);
5617	Assert(pVM->hm.s.vmx.pRealModeTSS);
5618	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
5619	if ( pVmcsInfo->fWasInRealMode
5620	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
5621	{
5622	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
5623	in real-mode (e.g. OpenBSD 4.0) */
5624	REMFlushTBs(pVM);
5625	Log4Func(("Switch to protected mode detected!\n"));
5626	pVmcsInfo->fWasInRealMode = false;
5627	}
5628	}
5629	#endif
5630	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
5631	{
5632	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
5633	if (pVmcsInfo->RealMode.fRealOnV86Active)
5634	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
5635	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
5636	AssertRCReturn(rc, rc);
5637	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
5638	}
5639
5640	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
5641	{
5642	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
5643	if (pVmcsInfo->RealMode.fRealOnV86Active)
5644	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
5645	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
5646	AssertRCReturn(rc, rc);
5647	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
5648	}
5649
5650	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
5651	{
5652	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
5653	if (pVmcsInfo->RealMode.fRealOnV86Active)
5654	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
5655	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
5656	AssertRCReturn(rc, rc);
5657	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
5658	}
5659
5660	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
5661	{
5662	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
5663	if (pVmcsInfo->RealMode.fRealOnV86Active)
5664	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
5665	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
5666	AssertRCReturn(rc, rc);
5667	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
5668	}
5669
5670	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
5671	{
5672	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
5673	if (pVmcsInfo->RealMode.fRealOnV86Active)
5674	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
5675	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
5676	AssertRCReturn(rc, rc);
5677	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
5678	}
5679
5680	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
5681	{
5682	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
5683	if (pVmcsInfo->RealMode.fRealOnV86Active)
5684	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
5685	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
5686	AssertRCReturn(rc, rc);
5687	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
5688	}
5689
5690	#ifdef VBOX_STRICT
5691	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
5692	#endif
5693	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
5694	pCtx->cs.Attr.u));
5695	}
5696
5697	/*
5698	* Guest TR.
5699	*/
5700	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
5701	{
5702	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
5703
5704	/*
5705	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
5706	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
5707	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
5708	*/
5709	uint16_t u16Sel;
5710	uint32_t u32Limit;
5711	uint64_t u64Base;
5712	uint32_t u32AccessRights;
5713	if (!pVmcsInfo->RealMode.fRealOnV86Active)
5714	{
5715	u16Sel = pCtx->tr.Sel;
5716	u32Limit = pCtx->tr.u32Limit;
5717	u64Base = pCtx->tr.u64Base;
5718	u32AccessRights = pCtx->tr.Attr.u;
5719	}
5720	else
5721	{
5722	Assert(!pVmxTransient->fIsNestedGuest);
5723	Assert(pVM->hm.s.vmx.pRealModeTSS);
5724	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
5725
5726	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
5727	RTGCPHYS GCPhys;
5728	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
5729	AssertRCReturn(rc, rc);
5730
5731	X86DESCATTR DescAttr;
5732	DescAttr.u = 0;
5733	DescAttr.n.u1Present = 1;
5734	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
5735
5736	u16Sel = 0;
5737	u32Limit = HM_VTX_TSS_SIZE;
5738	u64Base = GCPhys;
5739	u32AccessRights = DescAttr.u;
5740	}
5741
5742	/* Validate. */
5743	Assert(!(u16Sel & RT_BIT(2)));
5744	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
5745	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
5746	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
5747	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
5748	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
5749	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
5750	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
5751	Assert( (u32Limit & 0xfff) == 0xfff
5752	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
5753	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
5754	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
5755
5756	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
5757	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
5758	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
5759	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
5760	AssertRCReturn(rc, rc);
5761
5762	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
5763	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
5764	}
5765
5766	/*
5767	* Guest GDTR.
5768	*/
5769	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
5770	{
5771	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
5772
5773	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
5774	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
5775	AssertRCReturn(rc, rc);
5776
5777	/* Validate. */
5778	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5779
5780	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
5781	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
5782	}
5783
5784	/*
5785	* Guest LDTR.
5786	*/
5787	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
5788	{
5789	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
5790
5791	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
5792	uint32_t u32Access;
5793	if ( !pVmxTransient->fIsNestedGuest
5794	&& !pCtx->ldtr.Attr.u)
5795	u32Access = X86DESCATTR_UNUSABLE;
5796	else
5797	u32Access = pCtx->ldtr.Attr.u;
5798
5799	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
5800	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
5801	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
5802	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
5803	AssertRCReturn(rc, rc);
5804
5805	/* Validate. */
5806	if (!(u32Access & X86DESCATTR_UNUSABLE))
5807	{
5808	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
5809	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
5810	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
5811	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
5812	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
5813	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
5814	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
5815	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
5816	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
5817	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
5818	}
5819
5820	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
5821	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
5822	}
5823
5824	/*
5825	* Guest IDTR.
5826	*/
5827	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
5828	{
5829	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
5830
5831	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
5832	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
5833	AssertRCReturn(rc, rc);
5834
5835	/* Validate. */
5836	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5837
5838	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
5839	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
5840	}
5841
5842	return VINF_SUCCESS;
5843	}
5844
5845
5846	/**
5847	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
5848	* areas.
5849	*
5850	* These MSRs will automatically be loaded to the host CPU on every successful
5851	* VM-entry and stored from the host CPU on every successful VM-exit.
5852	*
5853	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
5854	* actual host MSR values are not- updated here for performance reasons. See
5855	* hmR0VmxExportHostMsrs().
5856	*
5857	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
5858	*
5859	* @returns VBox status code.
5860	* @param pVCpu The cross context virtual CPU structure.
5861	* @param pVmxTransient The VMX-transient structure.
5862	*
5863	* @remarks No-long-jump zone!!!
5864	*/
5865	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5866	{
5867	AssertPtr(pVCpu);
5868	AssertPtr(pVmxTransient);
5869
5870	PVM pVM = pVCpu->CTX_SUFF(pVM);
5871	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5872
5873	/*
5874	* MSRs that we use the auto-load/store MSR area in the VMCS.
5875	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
5876	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
5877	*
5878	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
5879	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
5880	* emulation, nothing to do here.
5881	*/
5882	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
5883	{
5884	if ( !pVmxTransient->fIsNestedGuest
5885	&& pVM->hm.s.fAllow64BitGuests)
5886	{
5887	#if HC_ARCH_BITS == 32
5888	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
5889	Assert(!pVmxTransient->fIsNestedGuest);
5890
5891	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
5892	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
5893	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
5894	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
5895	AssertRCReturn(rc, rc);
5896	#endif
5897	}
5898	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
5899	}
5900
5901	/*
5902	* Guest Sysenter MSRs.
5903	*/
5904	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
5905	{
5906	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
5907
5908	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
5909	{
5910	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
5911	AssertRCReturn(rc, rc);
5912	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
5913	}
5914
5915	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
5916	{
5917	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
5918	AssertRCReturn(rc, rc);
5919	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
5920	}
5921
5922	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
5923	{
5924	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
5925	AssertRCReturn(rc, rc);
5926	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
5927	}
5928	}
5929
5930	/*
5931	* Guest/host EFER MSR.
5932	*/
5933	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
5934	{
5935	/* Whether we are using the VMCS to swap the EFER MSR must have been
5936	determined earlier while exporting VM-entry/VM-exit controls. */
5937	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
5938	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
5939
5940	if (hmR0VmxShouldSwapEferMsr(pVCpu))
5941	{
5942	/*
5943	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
5944	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
5945	*/
5946	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
5947	{
5948	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
5949	AssertRCReturn(rc, rc);
5950	}
5951	else
5952	{
5953	/*
5954	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
5955	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
5956	*/
5957	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
5958	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5959	AssertRCReturn(rc, rc);
5960	}
5961	}
5962	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
5963	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
5964
5965	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
5966	}
5967
5968	/*
5969	* Other MSRs.
5970	* Speculation Control (R/W).
5971	*/
5972	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
5973	{
5974	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
5975	if (pVM->cpum.ro.GuestFeatures.fIbrs)
5976	{
5977	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
5978	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5979	AssertRCReturn(rc, rc);
5980	}
5981	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
5982	}
5983
5984	return VINF_SUCCESS;
5985	}
5986
5987
5988	/**
5989	* Selects up the appropriate function to run guest code.
5990	*
5991	* @returns VBox status code.
5992	* @param pVCpu The cross context virtual CPU structure.
5993	* @param pVmxTransient The VMX-transient structure.
5994	*
5995	* @remarks No-long-jump zone!!!
5996	*/
5997	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5998	{
5999	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6000	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6001
6002	if (CPUMIsGuestInLongModeEx(pCtx))
6003	{
6004	#ifndef VBOX_ENABLE_64_BITS_GUESTS
6005	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
6006	#endif
6007	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
6008	#if HC_ARCH_BITS == 32
6009	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
6010	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
6011	{
6012	#ifdef VBOX_STRICT
6013	if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
6014	{
6015	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
6016	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
6017	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
6018	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
6019	("fCtxChanged=%#RX64\n", fCtxChanged));
6020	}
6021	#endif
6022	pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
6023
6024	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
6025	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
6026	pVmcsInfo->fSwitchedTo64on32 = true;
6027	Log4Func(("Selected 64-bit switcher\n"));
6028	}
6029	#else
6030	/* 64-bit host. */
6031	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
6032	#endif
6033	}
6034	else
6035	{
6036	/* Guest is not in long mode, use the 32-bit handler. */
6037	#if HC_ARCH_BITS == 32
6038	if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
6039	&& !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
6040	&& pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
6041	{
6042	# ifdef VBOX_STRICT
6043	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
6044	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
6045	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
6046	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
6047	("fCtxChanged=%#RX64\n", fCtxChanged));
6048	# endif
6049	}
6050	# ifdef VBOX_ENABLE_64_BITS_GUESTS
6051	/*
6052	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
6053	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
6054	* switcher flag now because we know the guest is in a sane state where it's safe
6055	* to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
6056	* the much faster 32-bit switcher again.
6057	*/
6058	if (!pVmcsInfo->fSwitchedTo64on32)
6059	{
6060	if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
6061	Log4Func(("Selected 32-bit switcher\n"));
6062	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6063	}
6064	else
6065	{
6066	Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
6067	if ( pVmcsInfo->RealMode.fRealOnV86Active
6068	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
6069	{
6070	pVmcsInfo->fSwitchedTo64on32 = false;
6071	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6072	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
6073	\| HM_CHANGED_VMX_ENTRY_EXIT_CTLS
6074	\| HM_CHANGED_HOST_CONTEXT);
6075	Log4Func(("Selected 32-bit switcher (safe)\n"));
6076	}
6077	}
6078	# else
6079	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6080	# endif
6081	#else
6082	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6083	#endif
6084	}
6085	Assert(pVmcsInfo->pfnStartVM);
6086	return VINF_SUCCESS;
6087	}
6088
6089
6090	/**
6091	* Wrapper for running the guest code in VT-x.
6092	*
6093	* @returns VBox status code, no informational status codes.
6094	* @param pVCpu The cross context virtual CPU structure.
6095	* @param pVmxTransient The VMX-transient structure.
6096	*
6097	* @remarks No-long-jump zone!!!
6098	*/
6099	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6100	{
6101	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6102	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6103	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6104
6105	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6106
6107	/*
6108	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6109	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6110	* callee-saved and thus the need for this XMM wrapper.
6111	*
6112	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6113	*/
6114	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6115	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6116	PVM pVM = pVCpu->CTX_SUFF(pVM);
6117	#ifdef VBOX_WITH_KERNEL_USING_XMM
6118	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6119	#else
6120	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
6121	#endif
6122	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6123	return rc;
6124	}
6125
6126
6127	/**
6128	* Reports world-switch error and dumps some useful debug info.
6129	*
6130	* @param pVCpu The cross context virtual CPU structure.
6131	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6132	* @param pVmxTransient The VMX-transient structure (only
6133	* exitReason updated).
6134	*/
6135	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6136	{
6137	Assert(pVCpu);
6138	Assert(pVmxTransient);
6139	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6140
6141	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6142	switch (rcVMRun)
6143	{
6144	case VERR_VMX_INVALID_VMXON_PTR:
6145	AssertFailed();
6146	break;
6147	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6148	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6149	{
6150	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6151	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6152	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
6153	AssertRC(rc);
6154
6155	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6156	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6157	Cannot do it here as we may have been long preempted. */
6158
6159	#ifdef VBOX_STRICT
6160	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6161	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6162	pVmxTransient->uExitReason));
6163	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6164	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6165	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6166	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6167	else
6168	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6169	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6170	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6171
6172	/* VMX control bits. */
6173	uint32_t u32Val;
6174	uint64_t u64Val;
6175	RTHCUINTREG uHCReg;
6176	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
6177	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
6178	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
6179	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
6180	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
6181	{
6182	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
6183	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
6184	}
6185	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
6186	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
6187	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
6188	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
6189	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
6190	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
6191	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
6192	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
6193	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
6194	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
6195	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
6196	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
6197	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
6198	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
6199	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
6200	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
6201	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6202	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
6203	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6204	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
6205	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
6206	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
6207	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
6208	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
6209	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
6210	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
6211	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
6212	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
6213	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
6214	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6215	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
6216	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
6217	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
6218	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6219	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6220	{
6221	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
6222	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
6223	}
6224
6225	/* Guest bits. */
6226	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
6227	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
6228	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
6229	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
6230	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
6231	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
6232	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
6233	{
6234	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
6235	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
6236	}
6237
6238	/* Host bits. */
6239	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
6240	Log4(("Host CR0 %#RHr\n", uHCReg));
6241	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
6242	Log4(("Host CR3 %#RHr\n", uHCReg));
6243	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
6244	Log4(("Host CR4 %#RHr\n", uHCReg));
6245
6246	RTGDTR HostGdtr;
6247	PCX86DESCHC pDesc;
6248	ASMGetGDTR(&HostGdtr);
6249	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
6250	Log4(("Host CS %#08x\n", u32Val));
6251	if (u32Val < HostGdtr.cbGdt)
6252	{
6253	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6254	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
6255	}
6256
6257	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
6258	Log4(("Host DS %#08x\n", u32Val));
6259	if (u32Val < HostGdtr.cbGdt)
6260	{
6261	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6262	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
6263	}
6264
6265	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
6266	Log4(("Host ES %#08x\n", u32Val));
6267	if (u32Val < HostGdtr.cbGdt)
6268	{
6269	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6270	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
6271	}
6272
6273	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
6274	Log4(("Host FS %#08x\n", u32Val));
6275	if (u32Val < HostGdtr.cbGdt)
6276	{
6277	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6278	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
6279	}
6280
6281	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
6282	Log4(("Host GS %#08x\n", u32Val));
6283	if (u32Val < HostGdtr.cbGdt)
6284	{
6285	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6286	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
6287	}
6288
6289	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
6290	Log4(("Host SS %#08x\n", u32Val));
6291	if (u32Val < HostGdtr.cbGdt)
6292	{
6293	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6294	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
6295	}
6296
6297	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
6298	Log4(("Host TR %#08x\n", u32Val));
6299	if (u32Val < HostGdtr.cbGdt)
6300	{
6301	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6302	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
6303	}
6304
6305	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
6306	Log4(("Host TR Base %#RHv\n", uHCReg));
6307	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
6308	Log4(("Host GDTR Base %#RHv\n", uHCReg));
6309	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
6310	Log4(("Host IDTR Base %#RHv\n", uHCReg));
6311	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
6312	Log4(("Host SYSENTER CS %#08x\n", u32Val));
6313	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
6314	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
6315	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
6316	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
6317	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
6318	Log4(("Host RSP %#RHv\n", uHCReg));
6319	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
6320	Log4(("Host RIP %#RHv\n", uHCReg));
6321	# if HC_ARCH_BITS == 64
6322	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6323	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6324	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6325	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6326	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6327	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6328	# endif
6329	#endif /* VBOX_STRICT */
6330	break;
6331	}
6332
6333	default:
6334	/* Impossible */
6335	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6336	break;
6337	}
6338	}
6339
6340
6341	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
6342	# ifndef VMX_USE_CACHED_VMCS_ACCESSES
6343	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
6344	# endif
6345
6346	/**
6347	* Initialize the VMCS-Read cache.
6348	*
6349	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
6350	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
6351	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
6352	* (those that have a 32-bit FULL & HIGH part).
6353	*
6354	* @param pVCpu The cross context virtual CPU structure.
6355	*/
6356	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
6357	{
6358	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
6359	do { \
6360	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
6361	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
6362	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
6363	++cReadFields; \
6364	} while (0)
6365
6366	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6367	uint32_t cReadFields = 0;
6368
6369	/*
6370	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
6371	* and serve to indicate exceptions to the rules.
6372	*/
6373
6374	/* Guest-natural selector base fields. */
6375	#if 0
6376	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
6377	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
6378	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
6379	#endif
6380	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
6381	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
6382	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
6383	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
6384	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
6385	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
6386	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
6387	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
6388	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
6389	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
6390	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
6391	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
6392	#if 0
6393	/* Unused natural width guest-state fields. */
6394	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
6395	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
6396	#endif
6397	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
6398	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
6399
6400	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
6401	these 64-bit fields (using "FULL" and "HIGH" fields). */
6402	#if 0
6403	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
6404	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
6405	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
6406	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
6407	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
6408	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
6409	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
6410	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
6411	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
6412	#endif
6413
6414	/* Natural width guest-state fields. */
6415	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
6416	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
6417
6418	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6419	{
6420	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
6421	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
6422	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
6423	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
6424	}
6425	else
6426	{
6427	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
6428	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
6429	}
6430
6431	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
6432	}
6433
6434
6435	/**
6436	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
6437	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
6438	* darwin, running 64-bit guests).
6439	*
6440	* @returns VBox status code.
6441	* @param pVCpu The cross context virtual CPU structure.
6442	* @param idxField The VMCS field encoding.
6443	* @param u64Val 16, 32 or 64-bit value.
6444	*/
6445	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6446	{
6447	int rc;
6448	switch (idxField)
6449	{
6450	/*
6451	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
6452	*/
6453	/* 64-bit Control fields. */
6454	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
6455	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
6456	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
6457	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
6458	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
6459	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
6460	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
6461	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
6462	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
6463	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
6464	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
6465	case VMX_VMCS64_CTRL_EPTP_FULL:
6466	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
6467	/* 64-bit Guest-state fields. */
6468	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
6469	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
6470	case VMX_VMCS64_GUEST_PAT_FULL:
6471	case VMX_VMCS64_GUEST_EFER_FULL:
6472	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
6473	case VMX_VMCS64_GUEST_PDPTE0_FULL:
6474	case VMX_VMCS64_GUEST_PDPTE1_FULL:
6475	case VMX_VMCS64_GUEST_PDPTE2_FULL:
6476	case VMX_VMCS64_GUEST_PDPTE3_FULL:
6477	/* 64-bit Host-state fields. */
6478	case VMX_VMCS64_HOST_PAT_FULL:
6479	case VMX_VMCS64_HOST_EFER_FULL:
6480	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
6481	{
6482	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6483	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
6484	break;
6485	}
6486
6487	/*
6488	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
6489	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
6490	*/
6491	/* Natural-width Guest-state fields. */
6492	case VMX_VMCS_GUEST_CR3:
6493	case VMX_VMCS_GUEST_ES_BASE:
6494	case VMX_VMCS_GUEST_CS_BASE:
6495	case VMX_VMCS_GUEST_SS_BASE:
6496	case VMX_VMCS_GUEST_DS_BASE:
6497	case VMX_VMCS_GUEST_FS_BASE:
6498	case VMX_VMCS_GUEST_GS_BASE:
6499	case VMX_VMCS_GUEST_LDTR_BASE:
6500	case VMX_VMCS_GUEST_TR_BASE:
6501	case VMX_VMCS_GUEST_GDTR_BASE:
6502	case VMX_VMCS_GUEST_IDTR_BASE:
6503	case VMX_VMCS_GUEST_RSP:
6504	case VMX_VMCS_GUEST_RIP:
6505	case VMX_VMCS_GUEST_SYSENTER_ESP:
6506	case VMX_VMCS_GUEST_SYSENTER_EIP:
6507	{
6508	if (!(RT_HI_U32(u64Val)))
6509	{
6510	/* If this field is 64-bit, VT-x will zero out the top bits. */
6511	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6512	}
6513	else
6514	{
6515	/* Assert that only the 32->64 switcher case should ever come here. */
6516	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
6517	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
6518	}
6519	break;
6520	}
6521
6522	default:
6523	{
6524	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
6525	pVCpu->hm.s.u32HMError = idxField;
6526	rc = VERR_INVALID_PARAMETER;
6527	break;
6528	}
6529	}
6530	AssertRCReturn(rc, rc);
6531	return rc;
6532	}
6533
6534
6535	/**
6536	* Queue up a VMWRITE by using the VMCS write cache.
6537	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
6538	*
6539	* @param pVCpu The cross context virtual CPU structure.
6540	* @param idxField The VMCS field encoding.
6541	* @param u64Val 16, 32 or 64-bit value.
6542	*/
6543	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6544	{
6545	AssertPtr(pVCpu);
6546	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6547
6548	AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
6549	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
6550
6551	/* Make sure there are no duplicates. */
6552	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
6553	{
6554	if (pCache->Write.aField[i] == idxField)
6555	{
6556	pCache->Write.aFieldVal[i] = u64Val;
6557	return VINF_SUCCESS;
6558	}
6559	}
6560
6561	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
6562	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
6563	pCache->Write.cValidEntries++;
6564	return VINF_SUCCESS;
6565	}
6566	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
6567
6568
6569	/**
6570	* Sets up the usage of TSC-offsetting and updates the VMCS.
6571	*
6572	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6573	* VMX-preemption timer.
6574	*
6575	* @returns VBox status code.
6576	* @param pVCpu The cross context virtual CPU structure.
6577	* @param pVmxTransient The VMX-transient structure.
6578	*
6579	* @remarks No-long-jump zone!!!
6580	*/
6581	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6582	{
6583	bool fOffsettedTsc;
6584	bool fParavirtTsc;
6585	uint64_t uTscOffset;
6586	PVM pVM = pVCpu->CTX_SUFF(pVM);
6587	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6588
6589	if (pVM->hm.s.vmx.fUsePreemptTimer)
6590	{
6591	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6592
6593	/* Make sure the returned values have sane upper and lower boundaries. */
6594	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6595	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6596	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6597	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6598
6599	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6600	* preemption timers here. We probably need to clamp the preemption timer,
6601	* after converting the timer value to the host. */
6602	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6603	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6604	AssertRC(rc);
6605	}
6606	else
6607	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6608
6609	if (fParavirtTsc)
6610	{
6611	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6612	information before every VM-entry, hence disable it for performance sake. */
6613	#if 0
6614	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
6615	AssertRC(rc);
6616	#endif
6617	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
6618	}
6619
6620	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
6621	if ( fOffsettedTsc
6622	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
6623	{
6624	if (pVmxTransient->fIsNestedGuest)
6625	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
6626	if (pVmcsInfo->u64TscOffset != uTscOffset)
6627	{
6628	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
6629	AssertRC(rc);
6630	pVmcsInfo->u64TscOffset = uTscOffset;
6631	}
6632
6633	if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
6634	{
6635	uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
6636	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6637	AssertRC(rc);
6638	pVmcsInfo->u32ProcCtls = uProcCtls;
6639	}
6640	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
6641	}
6642	else
6643	{
6644	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
6645	if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
6646	{
6647	uProcCtls \|= VMX_PROC_CTLS_RDTSC_EXIT;
6648	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6649	AssertRC(rc);
6650	pVmcsInfo->u32ProcCtls = uProcCtls;
6651	}
6652	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
6653	}
6654	}
6655
6656
6657	/**
6658	* Gets the IEM exception flags for the specified vector and IDT vectoring /
6659	* VM-exit interruption info type.
6660	*
6661	* @returns The IEM exception flags.
6662	* @param uVector The event vector.
6663	* @param uVmxEventType The VMX event type.
6664	*
6665	* @remarks This function currently only constructs flags required for
6666	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
6667	* and CR2 aspects of an exception are not included).
6668	*/
6669	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
6670	{
6671	uint32_t fIemXcptFlags;
6672	switch (uVmxEventType)
6673	{
6674	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6675	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6676	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6677	break;
6678
6679	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6680	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6681	break;
6682
6683	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6684	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
6685	break;
6686
6687	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
6688	{
6689	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6690	if (uVector == X86_XCPT_BP)
6691	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
6692	else if (uVector == X86_XCPT_OF)
6693	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
6694	else
6695	{
6696	fIemXcptFlags = 0;
6697	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
6698	}
6699	break;
6700	}
6701
6702	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6703	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6704	break;
6705
6706	default:
6707	fIemXcptFlags = 0;
6708	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
6709	break;
6710	}
6711	return fIemXcptFlags;
6712	}
6713
6714
6715	/**
6716	* Sets an event as a pending event to be injected into the guest.
6717	*
6718	* @param pVCpu The cross context virtual CPU structure.
6719	* @param u32IntInfo The VM-entry interruption-information field.
6720	* @param cbInstr The VM-entry instruction length in bytes (for software
6721	* interrupts, exceptions and privileged software
6722	* exceptions).
6723	* @param u32ErrCode The VM-entry exception error code.
6724	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
6725	* page-fault.
6726	*/
6727	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
6728	RTGCUINTPTR GCPtrFaultAddress)
6729	{
6730	Assert(!pVCpu->hm.s.Event.fPending);
6731	pVCpu->hm.s.Event.fPending = true;
6732	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
6733	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
6734	pVCpu->hm.s.Event.cbInstr = cbInstr;
6735	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
6736	}
6737
6738
6739	/**
6740	* Sets an external interrupt as pending-for-injection into the VM.
6741	*
6742	* @param pVCpu The cross context virtual CPU structure.
6743	* @param u8Interrupt The external interrupt vector.
6744	*/
6745	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
6746	{
6747	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
6748	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6749	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6750	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6751	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6752	}
6753
6754
6755	/**
6756	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
6757	*
6758	* @param pVCpu The cross context virtual CPU structure.
6759	*/
6760	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
6761	{
6762	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
6763	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
6764	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6765	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6766	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6767	}
6768
6769
6770	/**
6771	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
6772	*
6773	* @param pVCpu The cross context virtual CPU structure.
6774	*/
6775	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
6776	{
6777	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
6778	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6779	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6780	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6781	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6782	}
6783
6784
6785	/**
6786	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
6787	*
6788	* @param pVCpu The cross context virtual CPU structure.
6789	*/
6790	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
6791	{
6792	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
6793	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6794	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6795	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6796	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6797	}
6798
6799
6800	/**
6801	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
6802	*
6803	* @param pVCpu The cross context virtual CPU structure.
6804	*/
6805	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
6806	{
6807	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
6808	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6809	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6810	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6811	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6812	}
6813
6814
6815	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6816	/**
6817	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
6818	*
6819	* @param pVCpu The cross context virtual CPU structure.
6820	* @param u32ErrCode The error code for the general-protection exception.
6821	*/
6822	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
6823	{
6824	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
6825	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6826	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6827	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6828	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6829	}
6830
6831
6832	/**
6833	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
6834	*
6835	* @param pVCpu The cross context virtual CPU structure.
6836	* @param u32ErrCode The error code for the stack exception.
6837	*/
6838	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
6839	{
6840	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
6841	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6842	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6843	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6844	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6845	}
6846
6847
6848	/**
6849	* Decodes the memory operand of an instruction that caused a VM-exit.
6850	*
6851	* The VM-exit qualification field provides the displacement field for memory
6852	* operand instructions, if any.
6853	*
6854	* @returns Strict VBox status code (i.e. informational status codes too).
6855	* @retval VINF_SUCCESS if the operand was successfully decoded.
6856	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
6857	* operand.
6858	* @param pVCpu The cross context virtual CPU structure.
6859	* @param uExitInstrInfo The VM-exit instruction information field.
6860	* @param enmMemAccess The memory operand's access type (read or write).
6861	* @param GCPtrDisp The instruction displacement field, if any. For
6862	* RIP-relative addressing pass RIP + displacement here.
6863	* @param pGCPtrMem Where to store the effective destination memory address.
6864	*
6865	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
6866	* virtual-8086 mode hence skips those checks while verifying if the
6867	* segment is valid.
6868	*/
6869	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
6870	PRTGCPTR pGCPtrMem)
6871	{
6872	Assert(pGCPtrMem);
6873	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
6874	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
6875	\| CPUMCTX_EXTRN_CR0);
6876
6877	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
6878	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
6879	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
6880
6881	VMXEXITINSTRINFO ExitInstrInfo;
6882	ExitInstrInfo.u = uExitInstrInfo;
6883	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
6884	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
6885	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
6886	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
6887	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
6888	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
6889	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
6890	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
6891	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
6892
6893	/*
6894	* Validate instruction information.
6895	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
6896	*/
6897	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
6898	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
6899	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
6900	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
6901	AssertLogRelMsgReturn(fIsMemOperand,
6902	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
6903
6904	/*
6905	* Compute the complete effective address.
6906	*
6907	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
6908	* See AMD spec. 4.5.2 "Segment Registers".
6909	*/
6910	RTGCPTR GCPtrMem = GCPtrDisp;
6911	if (fBaseRegValid)
6912	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
6913	if (fIdxRegValid)
6914	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
6915
6916	RTGCPTR const GCPtrOff = GCPtrMem;
6917	if ( !fIsLongMode
6918	\|\| iSegReg >= X86_SREG_FS)
6919	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
6920	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
6921
6922	/*
6923	* Validate effective address.
6924	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
6925	*/
6926	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
6927	Assert(cbAccess > 0);
6928	if (fIsLongMode)
6929	{
6930	if (X86_IS_CANONICAL(GCPtrMem))
6931	{
6932	*pGCPtrMem = GCPtrMem;
6933	return VINF_SUCCESS;
6934	}
6935
6936	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
6937	* "Data Limit Checks in 64-bit Mode". */
6938	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
6939	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6940	return VINF_HM_PENDING_XCPT;
6941	}
6942
6943	/*
6944	* This is a watered down version of iemMemApplySegment().
6945	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
6946	* and segment CPL/DPL checks are skipped.
6947	*/
6948	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
6949	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
6950	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
6951
6952	/* Check if the segment is present and usable. */
6953	if ( pSel->Attr.n.u1Present
6954	&& !pSel->Attr.n.u1Unusable)
6955	{
6956	Assert(pSel->Attr.n.u1DescType);
6957	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
6958	{
6959	/* Check permissions for the data segment. */
6960	if ( enmMemAccess == VMXMEMACCESS_WRITE
6961	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
6962	{
6963	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6964	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
6965	return VINF_HM_PENDING_XCPT;
6966	}
6967
6968	/* Check limits if it's a normal data segment. */
6969	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
6970	{
6971	if ( GCPtrFirst32 > pSel->u32Limit
6972	\|\| GCPtrLast32 > pSel->u32Limit)
6973	{
6974	Log4Func(("Data segment limit exceeded. "
6975	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6976	GCPtrLast32, pSel->u32Limit));
6977	if (iSegReg == X86_SREG_SS)
6978	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6979	else
6980	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6981	return VINF_HM_PENDING_XCPT;
6982	}
6983	}
6984	else
6985	{
6986	/* Check limits if it's an expand-down data segment.
6987	Note! The upper boundary is defined by the B bit, not the G bit! */
6988	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
6989	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
6990	{
6991	Log4Func(("Expand-down data segment limit exceeded. "
6992	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6993	GCPtrLast32, pSel->u32Limit));
6994	if (iSegReg == X86_SREG_SS)
6995	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6996	else
6997	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6998	return VINF_HM_PENDING_XCPT;
6999	}
7000	}
7001	}
7002	else
7003	{
7004	/* Check permissions for the code segment. */
7005	if ( enmMemAccess == VMXMEMACCESS_WRITE
7006	\|\| ( enmMemAccess == VMXMEMACCESS_READ
7007	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
7008	{
7009	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
7010	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
7011	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7012	return VINF_HM_PENDING_XCPT;
7013	}
7014
7015	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
7016	if ( GCPtrFirst32 > pSel->u32Limit
7017	\|\| GCPtrLast32 > pSel->u32Limit)
7018	{
7019	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
7020	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
7021	if (iSegReg == X86_SREG_SS)
7022	hmR0VmxSetPendingXcptSS(pVCpu, 0);
7023	else
7024	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7025	return VINF_HM_PENDING_XCPT;
7026	}
7027	}
7028	}
7029	else
7030	{
7031	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
7032	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7033	return VINF_HM_PENDING_XCPT;
7034	}
7035
7036	*pGCPtrMem = GCPtrMem;
7037	return VINF_SUCCESS;
7038	}
7039
7040
7041	/**
7042	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
7043	* guest attempting to execute a VMX instruction.
7044	*
7045	* @returns Strict VBox status code (i.e. informational status codes too).
7046	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
7047	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
7048	*
7049	* @param pVCpu The cross context virtual CPU structure.
7050	* @param uExitReason The VM-exit reason.
7051	*
7052	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
7053	* @remarks No-long-jump zone!!!
7054	*/
7055	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
7056	{
7057	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
7058	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
7059
7060	/*
7061	* The physical CPU would have already checked the CPU mode/code segment.
7062	* We shall just assert here for paranoia.
7063	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
7064	*/
7065	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
7066	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
7067	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
7068
7069	if (uExitReason == VMX_EXIT_VMXON)
7070	{
7071	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
7072
7073	/*
7074	* We check CR4.VMXE because it is required to be always set while in VMX operation
7075	* by physical CPUs and our CR4 read shadow is only consulted when executing specific
7076	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
7077	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
7078	*/
7079	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
7080	{
7081	Log4Func(("CR4.VMXE is not set -> #UD\n"));
7082	hmR0VmxSetPendingXcptUD(pVCpu);
7083	return VINF_HM_PENDING_XCPT;
7084	}
7085	}
7086	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
7087	{
7088	/*
7089	* The guest has not entered VMX operation but attempted to execute a VMX instruction
7090	* (other than VMXON), we need to raise a #UD.
7091	*/
7092	Log4Func(("Not in VMX root mode -> #UD\n"));
7093	hmR0VmxSetPendingXcptUD(pVCpu);
7094	return VINF_HM_PENDING_XCPT;
7095	}
7096
7097	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
7098	return VINF_SUCCESS;
7099	}
7100	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7101
7102
7103	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7104	{
7105	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7106
7107	/*
7108	* If VT-x marks the segment as unusable, most other bits remain undefined:
7109	* - For CS the L, D and G bits have meaning.
7110	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7111	* - For the remaining data segments no bits are defined.
7112	*
7113	* The present bit and the unusable bit has been observed to be set at the
7114	* same time (the selector was supposed to be invalid as we started executing
7115	* a V8086 interrupt in ring-0).
7116	*
7117	* What should be important for the rest of the VBox code, is that the P bit is
7118	* cleared. Some of the other VBox code recognizes the unusable bit, but
7119	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7120	* safe side here, we'll strip off P and other bits we don't care about. If
7121	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7122	*
7123	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7124	*/
7125	#ifdef VBOX_STRICT
7126	uint32_t const uAttr = pSelReg->Attr.u;
7127	#endif
7128
7129	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7130	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7131	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7132
7133	#ifdef VBOX_STRICT
7134	VMMRZCallRing3Disable(pVCpu);
7135	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7136	# ifdef DEBUG_bird
7137	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7138	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7139	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7140	# endif
7141	VMMRZCallRing3Enable(pVCpu);
7142	NOREF(uAttr);
7143	#endif
7144	RT_NOREF2(pVCpu, idxSel);
7145	}
7146
7147
7148	/**
7149	* Imports a guest segment register from the current VMCS into the guest-CPU
7150	* context.
7151	*
7152	* @returns VBox status code.
7153	* @param pVCpu The cross context virtual CPU structure.
7154	* @param iSegReg The segment register number (X86_SREG_XXX).
7155	*
7156	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7157	* do not log!
7158	*/
7159	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7160	{
7161	Assert(iSegReg < X86_SREG_COUNT);
7162
7163	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7164	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7165	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7166	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
7167	uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
7168	#else
7169	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7170	#endif
7171	uint64_t u64Base;
7172	uint32_t u32Sel, u32Limit, u32Attr;
7173	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7174	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7175	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7176	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
7177	if (RT_SUCCESS(rc))
7178	{
7179	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7180	pSelReg->Sel = u32Sel;
7181	pSelReg->ValidSel = u32Sel;
7182	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7183	pSelReg->u32Limit = u32Limit;
7184	pSelReg->u64Base = u64Base;
7185	pSelReg->Attr.u = u32Attr;
7186	if (u32Attr & X86DESCATTR_UNUSABLE)
7187	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7188	}
7189	return rc;
7190	}
7191
7192
7193	/**
7194	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7195	*
7196	* @returns VBox status code.
7197	* @param pVCpu The cross context virtual CPU structure.
7198	*
7199	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7200	* do not log!
7201	*/
7202	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7203	{
7204	uint64_t u64Base;
7205	uint32_t u32Sel, u32Limit, u32Attr;
7206	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7207	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7208	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7209	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7210	if (RT_SUCCESS(rc))
7211	{
7212	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7213	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7214	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7215	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7216	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7217	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7218	if (u32Attr & X86DESCATTR_UNUSABLE)
7219	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7220	}
7221	return rc;
7222	}
7223
7224
7225	/**
7226	* Imports the guest TR from the current VMCS into the guest-CPU context.
7227	*
7228	* @returns VBox status code.
7229	* @param pVCpu The cross context virtual CPU structure.
7230	*
7231	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7232	* do not log!
7233	*/
7234	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7235	{
7236	uint32_t u32Sel, u32Limit, u32Attr;
7237	uint64_t u64Base;
7238	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7239	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7240	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7241	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7242	AssertRCReturn(rc, rc);
7243
7244	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7245	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7246	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7247	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7248	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7249	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7250	/* TR is the only selector that can never be unusable. */
7251	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7252	return VINF_SUCCESS;
7253	}
7254
7255
7256	/**
7257	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7258	*
7259	* @returns VBox status code.
7260	* @param pVCpu The cross context virtual CPU structure.
7261	*
7262	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7263	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7264	* instead!!!
7265	*/
7266	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7267	{
7268	uint64_t u64Val;
7269	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7270	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7271	{
7272	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7273	if (RT_SUCCESS(rc))
7274	{
7275	pCtx->rip = u64Val;
7276	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7277	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7278	}
7279	return rc;
7280	}
7281	return VINF_SUCCESS;
7282	}
7283
7284
7285	/**
7286	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7287	*
7288	* @returns VBox status code.
7289	* @param pVCpu The cross context virtual CPU structure.
7290	* @param pVmcsInfo The VMCS info. object.
7291	*
7292	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7293	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7294	* instead!!!
7295	*/
7296	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7297	{
7298	uint32_t u32Val;
7299	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7300	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7301	{
7302	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7303	if (RT_SUCCESS(rc))
7304	{
7305	pCtx->eflags.u32 = u32Val;
7306
7307	/* Restore eflags for real-on-v86-mode hack. */
7308	if (pVmcsInfo->RealMode.fRealOnV86Active)
7309	{
7310	pCtx->eflags.Bits.u1VM = 0;
7311	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7312	}
7313	}
7314	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7315	return rc;
7316	}
7317	return VINF_SUCCESS;
7318	}
7319
7320
7321	/**
7322	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7323	* context.
7324	*
7325	* @returns VBox status code.
7326	* @param pVCpu The cross context virtual CPU structure.
7327	* @param pVmcsInfo The VMCS info. object.
7328	*
7329	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7330	* do not log!
7331	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7332	* instead!!!
7333	*/
7334	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7335	{
7336	uint32_t u32Val;
7337	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7338	if (RT_SUCCESS(rc))
7339	{
7340	if (!u32Val)
7341	{
7342	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7343	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7344
7345	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7346	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7347	}
7348	else
7349	{
7350	/*
7351	* We must import RIP here to set our EM interrupt-inhibited state.
7352	* We also import RFLAGS as our code that evaluates pending interrupts
7353	* before VM-entry requires it.
7354	*/
7355	rc = hmR0VmxImportGuestRip(pVCpu);
7356	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7357	if (RT_SUCCESS(rc))
7358	{
7359	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7360	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7361	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7362	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7363
7364	if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
7365	{
7366	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7367	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
7368	}
7369	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7370	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7371	}
7372	}
7373	}
7374	return rc;
7375	}
7376
7377
7378	/**
7379	* Worker for VMXR0ImportStateOnDemand.
7380	*
7381	* @returns VBox status code.
7382	* @param pVCpu The cross context virtual CPU structure.
7383	* @param pVmcsInfo The VMCS info. object.
7384	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7385	*/
7386	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7387	{
7388	#define VMXLOCAL_BREAK_RC(a_rc) \
7389	if (RT_SUCCESS(a_rc)) \
7390	{ } \
7391	else \
7392	break
7393
7394	int rc = VINF_SUCCESS;
7395	PVM pVM = pVCpu->CTX_SUFF(pVM);
7396	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7397	uint64_t u64Val;
7398	uint32_t u32Val;
7399
7400	/*
7401	* Note! This is hack to workaround a mysterious BSOD observed with release builds
7402	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
7403	* neither are other host platforms.
7404	*
7405	* Committing this temporarily as it prevents BSOD.
7406	*
7407	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
7408	*/
7409	#ifdef RT_OS_WINDOWS
7410	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
7411	return VERR_HM_IPE_1;
7412	#endif
7413
7414	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
7415
7416	/*
7417	* We disable interrupts to make the updating of the state and in particular
7418	* the fExtrn modification atomic wrt to preemption hooks.
7419	*/
7420	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7421
7422	fWhat &= pCtx->fExtrn;
7423	if (fWhat)
7424	{
7425	do
7426	{
7427	if (fWhat & CPUMCTX_EXTRN_RIP)
7428	{
7429	rc = hmR0VmxImportGuestRip(pVCpu);
7430	VMXLOCAL_BREAK_RC(rc);
7431	}
7432
7433	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7434	{
7435	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7436	VMXLOCAL_BREAK_RC(rc);
7437	}
7438
7439	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7440	{
7441	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7442	VMXLOCAL_BREAK_RC(rc);
7443	}
7444
7445	if (fWhat & CPUMCTX_EXTRN_RSP)
7446	{
7447	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
7448	VMXLOCAL_BREAK_RC(rc);
7449	pCtx->rsp = u64Val;
7450	}
7451
7452	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7453	{
7454	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7455	if (fWhat & CPUMCTX_EXTRN_CS)
7456	{
7457	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7458	rc \|= hmR0VmxImportGuestRip(pVCpu);
7459	if (fRealOnV86Active)
7460	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7461	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7462	}
7463	if (fWhat & CPUMCTX_EXTRN_SS)
7464	{
7465	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7466	if (fRealOnV86Active)
7467	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7468	}
7469	if (fWhat & CPUMCTX_EXTRN_DS)
7470	{
7471	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7472	if (fRealOnV86Active)
7473	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7474	}
7475	if (fWhat & CPUMCTX_EXTRN_ES)
7476	{
7477	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7478	if (fRealOnV86Active)
7479	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7480	}
7481	if (fWhat & CPUMCTX_EXTRN_FS)
7482	{
7483	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7484	if (fRealOnV86Active)
7485	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7486	}
7487	if (fWhat & CPUMCTX_EXTRN_GS)
7488	{
7489	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7490	if (fRealOnV86Active)
7491	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7492	}
7493	VMXLOCAL_BREAK_RC(rc);
7494	}
7495
7496	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7497	{
7498	if (fWhat & CPUMCTX_EXTRN_LDTR)
7499	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
7500
7501	if (fWhat & CPUMCTX_EXTRN_GDTR)
7502	{
7503	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
7504	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
7505	pCtx->gdtr.pGdt = u64Val;
7506	pCtx->gdtr.cbGdt = u32Val;
7507	}
7508
7509	/* Guest IDTR. */
7510	if (fWhat & CPUMCTX_EXTRN_IDTR)
7511	{
7512	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
7513	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
7514	pCtx->idtr.pIdt = u64Val;
7515	pCtx->idtr.cbIdt = u32Val;
7516	}
7517
7518	/* Guest TR. */
7519	if (fWhat & CPUMCTX_EXTRN_TR)
7520	{
7521	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7522	don't need to import that one. */
7523	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7524	rc \|= hmR0VmxImportGuestTr(pVCpu);
7525	}
7526	VMXLOCAL_BREAK_RC(rc);
7527	}
7528
7529	if (fWhat & CPUMCTX_EXTRN_DR7)
7530	{
7531	if (!pVCpu->hm.s.fUsingHyperDR7)
7532	{
7533	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
7534	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
7535	VMXLOCAL_BREAK_RC(rc);
7536	pCtx->dr[7] = u32Val;
7537	}
7538	}
7539
7540	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7541	{
7542	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
7543	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
7544	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
7545	pCtx->SysEnter.cs = u32Val;
7546	VMXLOCAL_BREAK_RC(rc);
7547	}
7548
7549	#if HC_ARCH_BITS == 64
7550	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7551	{
7552	if ( pVM->hm.s.fAllow64BitGuests
7553	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7554	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7555	}
7556
7557	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7558	{
7559	if ( pVM->hm.s.fAllow64BitGuests
7560	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7561	{
7562	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7563	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7564	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7565	}
7566	}
7567	#endif
7568
7569	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7570	#if HC_ARCH_BITS == 32
7571	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
7572	#endif
7573	)
7574	{
7575	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7576	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7577	Assert(pMsrs);
7578	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
7579	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
7580	for (uint32_t i = 0; i < cMsrs; i++)
7581	{
7582	uint32_t const idMsr = pMsrs[i].u32Msr;
7583	switch (idMsr)
7584	{
7585	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
7586	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
7587	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7588	#if HC_ARCH_BITS == 32
7589	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsrs[i].u64Value; break;
7590	case MSR_K6_STAR: pCtx->msrSTAR = pMsrs[i].u64Value; break;
7591	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsrs[i].u64Value; break;
7592	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsrs[i].u64Value; break;
7593	#endif
7594	default:
7595	{
7596	pCtx->fExtrn = 0;
7597	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
7598	ASMSetFlags(fEFlags);
7599	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
7600	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7601	}
7602	}
7603	}
7604	}
7605
7606	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7607	{
7608	uint64_t u64Shadow;
7609	if (fWhat & CPUMCTX_EXTRN_CR0)
7610	{
7611	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7612	* remove when we drop 32-bit host w/ 64-bit host support, see
7613	* @bugref{9180#c39}. */
7614	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7615	#if HC_ARCH_BITS == 32
7616	uint32_t u32Shadow;
7617	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
7618	u64Shadow = u32Shadow;
7619	#else
7620	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
7621	#endif
7622	VMXLOCAL_BREAK_RC(rc);
7623	u64Val = u32Val;
7624	#if 1
7625	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7626	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7627	#else
7628	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
7629	{
7630	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7631	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7632	}
7633	else
7634	{
7635	/** @todo NSTVMX: We need to do some unfudging here because we altered the
7636	* guest/host mask before running the nested-guest. */
7637	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
7638	Assert(pVmcsNstGst);
7639
7640	uint64_t const uGstCr0Mask = pVmcsNstGst->u64Cr0Mask.u;
7641	uint64_t const uHstCr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
7642	}
7643	#endif
7644	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7645	CPUMSetGuestCR0(pVCpu, u64Val);
7646	VMMRZCallRing3Enable(pVCpu);
7647	}
7648
7649	if (fWhat & CPUMCTX_EXTRN_CR4)
7650	{
7651	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7652	* remove when we drop 32-bit host w/ 64-bit host support, see
7653	* @bugref{9180#c39}. */
7654	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7655	#if HC_ARCH_BITS == 32
7656	uint32_t u32Shadow;
7657	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
7658	u64Shadow = u32Shadow;
7659	#else
7660	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
7661	#endif
7662	VMXLOCAL_BREAK_RC(rc);
7663	u64Val = u32Val;
7664	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
7665	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7666	pCtx->cr4 = u64Val;
7667	}
7668
7669	if (fWhat & CPUMCTX_EXTRN_CR3)
7670	{
7671	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7672	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7673	\|\| ( pVM->hm.s.fNestedPaging
7674	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7675	{
7676	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
7677	VMXLOCAL_BREAK_RC(rc);
7678	if (pCtx->cr3 != u64Val)
7679	{
7680	pCtx->cr3 = u64Val;
7681	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7682	}
7683
7684	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7685	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7686	if (CPUMIsGuestInPAEModeEx(pCtx))
7687	{
7688	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
7689	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
7690	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
7691	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
7692	VMXLOCAL_BREAK_RC(rc);
7693	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7694	}
7695	}
7696	}
7697
7698	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7699	# if 0
7700	/** @todo NSTVMX: We handle most of these fields individually by passing it to IEM
7701	* VM-exit handlers as parameters. We would handle it differently when using
7702	* the fast path. */
7703	/*
7704	* The hardware virtualization state currently consists of VMCS fields that may be
7705	* modified by execution of the nested-guest (that are not part of the general
7706	* guest state) and is visible to guest software. Hence, it is technically part of
7707	* the guest-CPU state when executing a nested-guest.
7708	*/
7709	if ( (fWhat & CPUMCTX_EXTRN_HWVIRT)
7710	&& CPUMIsGuestInVmxNonRootMode(pCtx))
7711	{
7712	PVMXVVMCS pGstVmcs = pCtx->hwvirt.vmx.CTX_SUFF(pVmcs);
7713	rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pGstVmcs->u32RoExitReason);
7714	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pGstVmcs->u64RoExitQual.u);
7715	VMXLOCAL_BREAK_RC(rc);
7716
7717	/*
7718	* VM-entry can fail due to invalid-guest state, machine-check events and
7719	* MSR loading failures. Other than VM-exit reason and VM-exit qualification
7720	* all other VMCS fields are left unmodified on VM-entry failure.
7721	*
7722	* See Intel spec. 26.7 "VM-entry Failures During Or After Loading Guest State".
7723	*/
7724	bool const fEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(pGstVmcs->u32RoExitReason);
7725	if (!fEntryFailed)
7726	{
7727	/*
7728	* Some notes on VMCS fields that may need importing when the fast path
7729	* is implemented. Currently we fully emulate VMLAUNCH/VMRESUME in IEM.
7730	*
7731	* Requires fixing up when using hardware-assisted VMX:
7732	* - VM-exit interruption info: Shouldn't reflect host interrupts/NMIs.
7733	* - VM-exit interruption error code: Cleared to 0 when not appropriate.
7734	* - IDT-vectoring info: Think about this.
7735	* - IDT-vectoring error code: Think about this.
7736	*
7737	* Emulated:
7738	* - Guest-interruptiblity state: Derived from FFs and RIP.
7739	* - Guest pending debug exceptions: Derived from DR6.
7740	* - Guest activity state: Emulated from EM state.
7741	* - Guest PDPTEs: Currently all 0s since we don't support nested EPT.
7742	* - Entry-interrupt info: Emulated, cleared to 0.
7743	*/
7744	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pGstVmcs->u32RoExitIntInfo);
7745	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pGstVmcs->u32RoExitIntErrCode);
7746	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pGstVmcs->u32RoIdtVectoringInfo);
7747	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pGstVmcs->u32RoIdtVectoringErrCode);
7748	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pGstVmcs->u32RoExitInstrLen);
7749	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pGstVmcs->u32RoExitIntInfo);
7750	rc \|= VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pGstVmcs->u64RoGuestPhysAddr.u);
7751	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pGstVmcs->u64RoGuestLinearAddr.u);
7752	/** @todo NSTVMX: Save and adjust preemption timer value. */
7753	}
7754
7755	VMXLOCAL_BREAK_RC(rc);
7756	}
7757	# endif
7758	#endif
7759	}
7760	} while (0);
7761
7762	if (RT_SUCCESS(rc))
7763	{
7764	/* Update fExtrn. */
7765	pCtx->fExtrn &= ~fWhat;
7766
7767	/* If everything has been imported, clear the HM keeper bit. */
7768	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7769	{
7770	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7771	Assert(!pCtx->fExtrn);
7772	}
7773	}
7774	}
7775	else
7776	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7777
7778	ASMSetFlags(fEFlags);
7779
7780	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7781
7782	if (RT_SUCCESS(rc))
7783	{ /* likely */ }
7784	else
7785	return rc;
7786
7787	/*
7788	* Honor any pending CR3 updates.
7789	*
7790	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
7791	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7792	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7793	*
7794	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7795	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7796	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7797	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7798	*
7799	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7800	*/
7801	if (VMMRZCallRing3IsEnabled(pVCpu))
7802	{
7803	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7804	{
7805	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7806	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7807	}
7808
7809	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7810	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7811
7812	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7813	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7814	}
7815
7816	return VINF_SUCCESS;
7817	#undef VMXLOCAL_BREAK_RC
7818	}
7819
7820
7821	/**
7822	* Saves the guest state from the VMCS into the guest-CPU context.
7823	*
7824	* @returns VBox status code.
7825	* @param pVCpu The cross context virtual CPU structure.
7826	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7827	*/
7828	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
7829	{
7830	PCVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7831	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7832	}
7833
7834
7835	/**
7836	* Check per-VM and per-VCPU force flag actions that require us to go back to
7837	* ring-3 for one reason or another.
7838	*
7839	* @returns Strict VBox status code (i.e. informational status codes too)
7840	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7841	* ring-3.
7842	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7843	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7844	* interrupts)
7845	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7846	* all EMTs to be in ring-3.
7847	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7848	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7849	* to the EM loop.
7850	*
7851	* @param pVCpu The cross context virtual CPU structure.
7852	* @param fStepping Whether we are single-stepping the guest using the
7853	* hypervisor debugger.
7854	*/
7855	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
7856	{
7857	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7858
7859	/*
7860	* Update pending interrupts into the APIC's IRR.
7861	*/
7862	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7863	APICUpdatePendingInterrupts(pVCpu);
7864
7865	/*
7866	* Anything pending? Should be more likely than not if we're doing a good job.
7867	*/
7868	PVM pVM = pVCpu->CTX_SUFF(pVM);
7869	if ( !fStepping
7870	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7871	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7872	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7873	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7874	return VINF_SUCCESS;
7875
7876	/* Pending PGM C3 sync. */
7877	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7878	{
7879	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7880	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7881	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7882	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7883	if (rcStrict2 != VINF_SUCCESS)
7884	{
7885	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7886	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7887	return rcStrict2;
7888	}
7889	}
7890
7891	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7892	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7893	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7894	{
7895	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7896	int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7897	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7898	return rc2;
7899	}
7900
7901	/* Pending VM request packets, such as hardware interrupts. */
7902	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7903	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7904	{
7905	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7906	return VINF_EM_PENDING_REQUEST;
7907	}
7908
7909	/* Pending PGM pool flushes. */
7910	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7911	{
7912	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7913	return VINF_PGM_POOL_FLUSH_PENDING;
7914	}
7915
7916	/* Pending DMA requests. */
7917	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7918	{
7919	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7920	return VINF_EM_RAW_TO_R3;
7921	}
7922
7923	return VINF_SUCCESS;
7924	}
7925
7926
7927	/**
7928	* Converts any TRPM trap into a pending HM event. This is typically used when
7929	* entering from ring-3 (not longjmp returns).
7930	*
7931	* @param pVCpu The cross context virtual CPU structure.
7932	*/
7933	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7934	{
7935	Assert(TRPMHasTrap(pVCpu));
7936	Assert(!pVCpu->hm.s.Event.fPending);
7937
7938	uint8_t uVector;
7939	TRPMEVENT enmTrpmEvent;
7940	RTGCUINT uErrCode;
7941	RTGCUINTPTR GCPtrFaultAddress;
7942	uint8_t cbInstr;
7943
7944	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7945	AssertRC(rc);
7946
7947	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7948	uint32_t u32IntInfo = uVector \| VMX_EXIT_INT_INFO_VALID;
7949	if (enmTrpmEvent == TRPM_TRAP)
7950	{
7951	/** @todo r=ramshankar: TRPM currently offers no way to determine a \#DB that was
7952	* generated using INT1 (ICEBP). */
7953	switch (uVector)
7954	{
7955	case X86_XCPT_NMI:
7956	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7957	break;
7958
7959	case X86_XCPT_BP:
7960	case X86_XCPT_OF:
7961	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7962	break;
7963
7964	case X86_XCPT_PF:
7965	case X86_XCPT_DF:
7966	case X86_XCPT_TS:
7967	case X86_XCPT_NP:
7968	case X86_XCPT_SS:
7969	case X86_XCPT_GP:
7970	case X86_XCPT_AC:
7971	u32IntInfo \|= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7972	RT_FALL_THRU();
7973	default:
7974	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7975	break;
7976	}
7977	}
7978	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7979	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7980	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7981	{
7982	switch (uVector)
7983	{
7984	case X86_XCPT_BP:
7985	case X86_XCPT_OF:
7986	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7987	break;
7988
7989	default:
7990	Assert(uVector == X86_XCPT_DB);
7991	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7992	break;
7993	}
7994	}
7995	else
7996	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7997
7998	rc = TRPMResetTrap(pVCpu);
7999	AssertRC(rc);
8000	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
8001	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
8002
8003	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
8004	}
8005
8006
8007	/**
8008	* Converts the pending HM event into a TRPM trap.
8009	*
8010	* @param pVCpu The cross context virtual CPU structure.
8011	*/
8012	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
8013	{
8014	Assert(pVCpu->hm.s.Event.fPending);
8015
8016	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8017	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
8018	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
8019	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
8020
8021	/* If a trap was already pending, we did something wrong! */
8022	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
8023
8024	/** @todo Use HMVmxEventToTrpmEventType() later. */
8025	TRPMEVENT enmTrapType;
8026	switch (uVectorType)
8027	{
8028	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
8029	enmTrapType = TRPM_HARDWARE_INT;
8030	break;
8031
8032	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
8033	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
8034	enmTrapType = TRPM_TRAP;
8035	break;
8036
8037	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT: /* #DB (INT1/ICEBP). */
8038	Assert(uVector == X86_XCPT_DB);
8039	enmTrapType = TRPM_SOFTWARE_INT;
8040	break;
8041
8042	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP (INT3) and #OF (INTO) */
8043	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8044	enmTrapType = TRPM_SOFTWARE_INT;
8045	break;
8046
8047	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
8048	enmTrapType = TRPM_SOFTWARE_INT;
8049	break;
8050
8051	default:
8052	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
8053	enmTrapType = TRPM_32BIT_HACK;
8054	break;
8055	}
8056
8057	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
8058
8059	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
8060	AssertRC(rc);
8061
8062	if (fErrorCodeValid)
8063	TRPMSetErrorCode(pVCpu, uErrorCode);
8064
8065	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8066	&& uVector == X86_XCPT_PF)
8067	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
8068	else if (enmTrapType == TRPM_SOFTWARE_INT)
8069	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
8070
8071	/* We're now done converting the pending event. */
8072	pVCpu->hm.s.Event.fPending = false;
8073	}
8074
8075
8076	/**
8077	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
8078	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
8079	*
8080	* @param pVCpu The cross context virtual CPU structure.
8081	* @param pVmcsInfo The VMCS info. object.
8082	*/
8083	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8084	{
8085	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8086	{
8087	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
8088	{
8089	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
8090	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8091	AssertRC(rc);
8092	}
8093	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
8094	}
8095
8096
8097	/**
8098	* Clears the interrupt-window exiting control in the VMCS.
8099	*
8100	* @param pVmcsInfo The VMCS info. object.
8101	*/
8102	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8103	{
8104	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8105	{
8106	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
8107	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8108	}
8109	return VINF_SUCCESS;
8110	}
8111
8112
8113	/**
8114	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
8115	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
8116	*
8117	* @param pVCpu The cross context virtual CPU structure.
8118	* @param pVmcsInfo The VMCS info. object.
8119	*/
8120	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8121	{
8122	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8123	{
8124	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
8125	{
8126	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8127	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8128	AssertRC(rc);
8129	Log4Func(("Setup NMI-window exiting\n"));
8130	}
8131	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
8132	}
8133
8134
8135	/**
8136	* Clears the NMI-window exiting control in the VMCS.
8137	*
8138	* @param pVmcsInfo The VMCS info. object.
8139	*/
8140	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8141	{
8142	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8143	{
8144	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8145	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8146	}
8147	return VINF_SUCCESS;
8148	}
8149
8150
8151	/**
8152	* Does the necessary state syncing before returning to ring-3 for any reason
8153	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
8154	*
8155	* @returns VBox status code.
8156	* @param pVCpu The cross context virtual CPU structure.
8157	* @param fImportState Whether to import the guest state from the VMCS back
8158	* to the guest-CPU context.
8159	*
8160	* @remarks No-long-jmp zone!!!
8161	*/
8162	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
8163	{
8164	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8165	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8166
8167	RTCPUID idCpu = RTMpCpuId();
8168	Log4Func(("HostCpuId=%u\n", idCpu));
8169
8170	/*
8171	* !!! IMPORTANT !!!
8172	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
8173	*/
8174
8175	/* Save the guest state if necessary. */
8176	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8177	if (fImportState)
8178	{
8179	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8180	AssertRCReturn(rc, rc);
8181	}
8182
8183	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8184	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8185	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8186
8187	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8188	#ifdef VBOX_STRICT
8189	if (CPUMIsHyperDebugStateActive(pVCpu))
8190	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8191	#endif
8192	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8193	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8194	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8195
8196	#if HC_ARCH_BITS == 64
8197	/* Restore host-state bits that VT-x only restores partially. */
8198	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8199	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8200	{
8201	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8202	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8203	}
8204	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8205	#endif
8206
8207	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8208	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8209	{
8210	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8211	if (!fImportState)
8212	{
8213	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8214	AssertRCReturn(rc, rc);
8215	}
8216	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8217	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8218	}
8219	else
8220	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8221
8222	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8223	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8224
8225	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8226	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8227	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8228	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8229	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8230	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8231	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8232	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8233	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8234
8235	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8236
8237	/** @todo This partially defeats the purpose of having preemption hooks.
8238	* The problem is, deregistering the hooks should be moved to a place that
8239	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8240	* context.
8241	*/
8242	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8243	AssertRCReturn(rc, rc);
8244
8245	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8246	NOREF(idCpu);
8247	return VINF_SUCCESS;
8248	}
8249
8250
8251	/**
8252	* Leaves the VT-x session.
8253	*
8254	* @returns VBox status code.
8255	* @param pVCpu The cross context virtual CPU structure.
8256	*
8257	* @remarks No-long-jmp zone!!!
8258	*/
8259	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8260	{
8261	HM_DISABLE_PREEMPT(pVCpu);
8262	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8263	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8264	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8265
8266	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8267	and done this from the VMXR0ThreadCtxCallback(). */
8268	if (!pVCpu->hm.s.fLeaveDone)
8269	{
8270	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8271	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8272	pVCpu->hm.s.fLeaveDone = true;
8273	}
8274	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8275
8276	/*
8277	* !!! IMPORTANT !!!
8278	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8279	*/
8280
8281	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8282	/** @todo Deregistering here means we need to VMCLEAR always
8283	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8284	* for calling VMMR0ThreadCtxHookDisable here! */
8285	VMMR0ThreadCtxHookDisable(pVCpu);
8286
8287	/* Leave HM context. This takes care of local init (term). */
8288	int rc = HMR0LeaveCpu(pVCpu);
8289
8290	HM_RESTORE_PREEMPT();
8291	return rc;
8292	}
8293
8294
8295	/**
8296	* Does the necessary state syncing before doing a longjmp to ring-3.
8297	*
8298	* @returns VBox status code.
8299	* @param pVCpu The cross context virtual CPU structure.
8300	*
8301	* @remarks No-long-jmp zone!!!
8302	*/
8303	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8304	{
8305	return hmR0VmxLeaveSession(pVCpu);
8306	}
8307
8308
8309	/**
8310	* Take necessary actions before going back to ring-3.
8311	*
8312	* An action requires us to go back to ring-3. This function does the necessary
8313	* steps before we can safely return to ring-3. This is not the same as longjmps
8314	* to ring-3, this is voluntary and prepares the guest so it may continue
8315	* executing outside HM (recompiler/IEM).
8316	*
8317	* @returns VBox status code.
8318	* @param pVCpu The cross context virtual CPU structure.
8319	* @param rcExit The reason for exiting to ring-3. Can be
8320	* VINF_VMM_UNKNOWN_RING3_CALL.
8321	*/
8322	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8323	{
8324	Assert(pVCpu);
8325	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8326
8327	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8328	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8329	{
8330	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8331	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8332	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8333	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8334	}
8335
8336	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8337	VMMRZCallRing3Disable(pVCpu);
8338	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8339
8340	/*
8341	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8342	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8343	*
8344	* This is because execution may continue from ring-3 and we would need to inject
8345	* the event from there (hence place it back in TRPM).
8346	*/
8347	if (pVCpu->hm.s.Event.fPending)
8348	{
8349	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8350	Assert(!pVCpu->hm.s.Event.fPending);
8351
8352	/* Clear the events from the VMCS. */
8353	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8354	AssertRCReturn(rc, rc);
8355	}
8356	#ifdef VBOX_STRICT
8357	else
8358	{
8359	/*
8360	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8361	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8362	* occasionally, see @bugref{9180#c42}.
8363	*/
8364	uint32_t uEntryIntInfo;
8365	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8366	AssertRC(rc);
8367	Assert(!VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8368	}
8369	#endif
8370
8371	/*
8372	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8373	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8374	* (e.g. TPR below threshold).
8375	*/
8376	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8377	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8378	AssertRCReturn(rc, rc);
8379
8380	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8381	and if we're injecting an event we should have a TRPM trap pending. */
8382	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8383	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8384	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8385	#endif
8386
8387	/* Save guest state and restore host state bits. */
8388	rc = hmR0VmxLeaveSession(pVCpu);
8389	AssertRCReturn(rc, rc);
8390	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8391
8392	/* Thread-context hooks are unregistered at this point!!! */
8393
8394	/* Sync recompiler state. */
8395	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8396	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8397	\| CPUM_CHANGED_LDTR
8398	\| CPUM_CHANGED_GDTR
8399	\| CPUM_CHANGED_IDTR
8400	\| CPUM_CHANGED_TR
8401	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8402	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8403	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8404	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8405
8406	Assert(!pVCpu->hm.s.fClearTrapFlag);
8407
8408	/* Update the exit-to-ring 3 reason. */
8409	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8410
8411	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8412	if ( rcExit != VINF_EM_RAW_INTERRUPT
8413	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8414	{
8415	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8416	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8417	}
8418
8419	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8420
8421	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
8422	VMMRZCallRing3RemoveNotification(pVCpu);
8423	VMMRZCallRing3Enable(pVCpu);
8424
8425	return rc;
8426	}
8427
8428
8429	/**
8430	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8431	* longjump to ring-3 and possibly get preempted.
8432	*
8433	* @returns VBox status code.
8434	* @param pVCpu The cross context virtual CPU structure.
8435	* @param enmOperation The operation causing the ring-3 longjump.
8436	* @param pvUser User argument, currently unused, NULL.
8437	*/
8438	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
8439	{
8440	RT_NOREF(pvUser);
8441	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8442	{
8443	/*
8444	* !!! IMPORTANT !!!
8445	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8446	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8447	*/
8448	VMMRZCallRing3RemoveNotification(pVCpu);
8449	VMMRZCallRing3Disable(pVCpu);
8450	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
8451	RTThreadPreemptDisable(&PreemptState);
8452
8453	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8454	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8455	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8456	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8457
8458	#if HC_ARCH_BITS == 64
8459	/* Restore host-state bits that VT-x only restores partially. */
8460	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8461	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8462	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8463	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8464	#endif
8465
8466	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8467	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8468	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8469
8470	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8471	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8472	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8473
8474	/* Clear the current VMCS data back to memory. */
8475	hmR0VmxClearVmcs(pVmcsInfo);
8476
8477	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8478	VMMR0ThreadCtxHookDisable(pVCpu);
8479	HMR0LeaveCpu(pVCpu);
8480	RTThreadPreemptRestore(&PreemptState);
8481	return VINF_SUCCESS;
8482	}
8483
8484	Assert(pVCpu);
8485	Assert(pvUser);
8486	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8487	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8488
8489	VMMRZCallRing3Disable(pVCpu);
8490	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8491
8492	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8493
8494	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8495	AssertRCReturn(rc, rc);
8496
8497	VMMRZCallRing3Enable(pVCpu);
8498	return VINF_SUCCESS;
8499	}
8500
8501
8502	/**
8503	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8504	* stack.
8505	*
8506	* @returns Strict VBox status code (i.e. informational status codes too).
8507	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8508	* @param pVCpu The cross context virtual CPU structure.
8509	* @param uValue The value to push to the guest stack.
8510	*/
8511	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
8512	{
8513	/*
8514	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8515	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8516	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8517	*/
8518	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8519	if (pCtx->sp == 1)
8520	return VINF_EM_RESET;
8521	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8522	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8523	AssertRC(rc);
8524	return rc;
8525	}
8526
8527
8528	/**
8529	* Injects an event into the guest upon VM-entry by updating the relevant fields
8530	* in the VM-entry area in the VMCS.
8531	*
8532	* @returns Strict VBox status code (i.e. informational status codes too).
8533	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8534	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8535	*
8536	* @param pVCpu The cross context virtual CPU structure.
8537	* @param pVmxTransient The VMX-transient structure.
8538	* @param pEvent The event being injected.
8539	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state.
8540	* This will be updated if necessary. This cannot not
8541	* be NULL.
8542	* @param fStepping Whether we're single-stepping guest execution and
8543	* should return VINF_EM_DBG_STEPPED if the event is
8544	* injected directly (registers modified by us, not by
8545	* hardware on VM-entry).
8546	*/
8547	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8548	uint32_t *pfIntrState)
8549	{
8550	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8551	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8552	Assert(pfIntrState);
8553
8554	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8555	uint32_t u32IntInfo = pEvent->u64IntInfo;
8556	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8557	uint32_t const cbInstr = pEvent->cbInstr;
8558	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8559	uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8560	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8561
8562	#ifdef VBOX_STRICT
8563	/*
8564	* Validate the error-code-valid bit for hardware exceptions.
8565	* No error codes for exceptions in real-mode.
8566	*
8567	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8568	*/
8569	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8570	&& !CPUMIsGuestInRealModeEx(pCtx))
8571	{
8572	switch (uVector)
8573	{
8574	case X86_XCPT_PF:
8575	case X86_XCPT_DF:
8576	case X86_XCPT_TS:
8577	case X86_XCPT_NP:
8578	case X86_XCPT_SS:
8579	case X86_XCPT_GP:
8580	case X86_XCPT_AC:
8581	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8582	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8583	RT_FALL_THRU();
8584	default:
8585	break;
8586	}
8587	}
8588
8589	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8590	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8591	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8592	#endif
8593
8594	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8595
8596	/*
8597	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8598	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8599	* interrupt handler in the (real-mode) guest.
8600	*
8601	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8602	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8603	*/
8604	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8605	{
8606	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8607	{
8608	/*
8609	* For CPUs with unrestricted guest execution enabled and with the guest
8610	* in real-mode, we must not set the deliver-error-code bit.
8611	*
8612	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8613	*/
8614	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8615	}
8616	else
8617	{
8618	PVM pVM = pVCpu->CTX_SUFF(pVM);
8619	Assert(PDMVmmDevHeapIsEnabled(pVM));
8620	Assert(pVM->hm.s.vmx.pRealModeTSS);
8621	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8622
8623	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8624	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8625	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8626	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8627	AssertRCReturn(rc2, rc2);
8628
8629	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8630	size_t const cbIdtEntry = sizeof(X86IDTR16);
8631	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8632	{
8633	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8634	if (uVector == X86_XCPT_DF)
8635	return VINF_EM_RESET;
8636
8637	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8638	No error codes for exceptions in real-mode. */
8639	if (uVector == X86_XCPT_GP)
8640	{
8641	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8642	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8643	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8644	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8645	HMEVENT EventXcptDf;
8646	RT_ZERO(EventXcptDf);
8647	EventXcptDf.u64IntInfo = uXcptDfInfo;
8648	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8649	}
8650
8651	/*
8652	* If we're injecting an event with no valid IDT entry, inject a #GP.
8653	* No error codes for exceptions in real-mode.
8654	*
8655	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8656	*/
8657	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8658	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8659	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8660	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8661	HMEVENT EventXcptGp;
8662	RT_ZERO(EventXcptGp);
8663	EventXcptGp.u64IntInfo = uXcptGpInfo;
8664	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8665	}
8666
8667	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8668	uint16_t uGuestIp = pCtx->ip;
8669	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8670	{
8671	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8672	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8673	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8674	}
8675	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8676	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8677
8678	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8679	X86IDTR16 IdtEntry;
8680	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8681	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8682	AssertRCReturn(rc2, rc2);
8683
8684	/* Construct the stack frame for the interrupt/exception handler. */
8685	VBOXSTRICTRC rcStrict;
8686	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8687	if (rcStrict == VINF_SUCCESS)
8688	{
8689	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8690	if (rcStrict == VINF_SUCCESS)
8691	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8692	}
8693
8694	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8695	if (rcStrict == VINF_SUCCESS)
8696	{
8697	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8698	pCtx->rip = IdtEntry.offSel;
8699	pCtx->cs.Sel = IdtEntry.uSel;
8700	pCtx->cs.ValidSel = IdtEntry.uSel;
8701	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8702	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8703	&& uVector == X86_XCPT_PF)
8704	pCtx->cr2 = GCPtrFault;
8705
8706	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8707	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8708	\| HM_CHANGED_GUEST_RSP);
8709
8710	/*
8711	* If we delivered a hardware exception (other than an NMI) and if there was
8712	* block-by-STI in effect, we should clear it.
8713	*/
8714	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8715	{
8716	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8717	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8718	Log4Func(("Clearing inhibition due to STI\n"));
8719	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8720	}
8721
8722	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8723	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8724
8725	/*
8726	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8727	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8728	*/
8729	pVCpu->hm.s.Event.fPending = false;
8730
8731	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8732	if (fStepping)
8733	rcStrict = VINF_EM_DBG_STEPPED;
8734	}
8735	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8736	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8737	return rcStrict;
8738	}
8739	}
8740
8741	/*
8742	* Validate.
8743	*/
8744	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8745	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8746
8747	/*
8748	* Inject the event into the VMCS.
8749	*/
8750	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8751	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8752	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8753	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8754	AssertRCReturn(rc, rc);
8755
8756	/*
8757	* Update guest CR2 if this is a page-fault.
8758	*/
8759	if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8760	&& uVector == X86_XCPT_PF)
8761	pCtx->cr2 = GCPtrFault;
8762
8763	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8764	return VINF_SUCCESS;
8765	}
8766
8767
8768	/**
8769	* Evaluates the event to be delivered to the guest and sets it as the pending
8770	* event.
8771	*
8772	* @returns Strict VBox status code (i.e. informational status codes too).
8773	* @param pVCpu The cross context virtual CPU structure.
8774	* @param pVmxTransient The VMX-transient structure.
8775	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8776	*/
8777	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8778	{
8779	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8780	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8781
8782	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
8783	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmcsInfo);
8784	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8785	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8786	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8787
8788	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8789	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8790	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8791	Assert(!TRPMHasTrap(pVCpu));
8792	Assert(pfIntrState);
8793
8794	*pfIntrState = fIntrState;
8795
8796	/*
8797	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
8798	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
8799	*/
8800	/** @todo SMI. SMIs take priority over NMIs. */
8801	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
8802	{
8803	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8804	if ( !pVCpu->hm.s.Event.fPending
8805	&& !fBlockNmi
8806	&& !fBlockSti
8807	&& !fBlockMovSS)
8808	{
8809	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8810	if ( pVmxTransient->fIsNestedGuest
8811	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8812	return IEMExecVmxVmexitNmi(pVCpu);
8813	#endif
8814	hmR0VmxSetPendingXcptNmi(pVCpu);
8815	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8816	Log4Func(("Pending NMI\n"));
8817	}
8818	else
8819	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8820	}
8821	/*
8822	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
8823	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
8824	*/
8825	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8826	&& !pVCpu->hm.s.fSingleInstruction)
8827	{
8828	Assert(!DBGFIsStepping(pVCpu));
8829	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8830	AssertRCReturn(rc, rc);
8831	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8832	if ( !pVCpu->hm.s.Event.fPending
8833	&& !fBlockInt
8834	&& !fBlockSti
8835	&& !fBlockMovSS)
8836	{
8837	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8838	if ( pVmxTransient->fIsNestedGuest
8839	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
8840	{
8841	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector /, true / fIntPending */);
8842	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8843	return rcStrict;
8844	}
8845	#endif
8846	uint8_t u8Interrupt;
8847	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8848	if (RT_SUCCESS(rc))
8849	{
8850	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8851	if ( pVmxTransient->fIsNestedGuest
8852	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8853	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8854	{
8855	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8856	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8857	return rcStrict;
8858	}
8859	#endif
8860	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8861	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
8862	}
8863	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8864	{
8865	if ( !pVmxTransient->fIsNestedGuest
8866	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8867	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
8868	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8869
8870	/*
8871	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8872	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8873	* need to re-set this force-flag here.
8874	*/
8875	}
8876	else
8877	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8878	}
8879	else
8880	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8881	}
8882
8883	return VINF_SUCCESS;
8884	}
8885
8886
8887	/**
8888	* Injects any pending events into the guest if the guest is in a state to
8889	* receive them.
8890	*
8891	* @returns Strict VBox status code (i.e. informational status codes too).
8892	* @param pVCpu The cross context virtual CPU structure.
8893	* @param pVmxTransient The VMX-transient structure.
8894	* @param fIntrState The VT-x guest-interruptibility state.
8895	* @param fStepping Whether we are single-stepping the guest using the
8896	* hypervisor debugger and should return
8897	* VINF_EM_DBG_STEPPED if the event was dispatched
8898	* directly.
8899	*/
8900	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
8901	{
8902	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8903	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8904
8905	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8906	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8907
8908	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8909	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8910	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8911	Assert(!TRPMHasTrap(pVCpu));
8912
8913	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8914	if (pVCpu->hm.s.Event.fPending)
8915	{
8916	/*
8917	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8918	* pending even while injecting an event and in this case, we want a VM-exit as soon as
8919	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
8920	*
8921	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8922	*/
8923	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8924	#ifdef VBOX_STRICT
8925	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8926	{
8927	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
8928	Assert(!fBlockInt);
8929	Assert(!fBlockSti);
8930	Assert(!fBlockMovSS);
8931	}
8932	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
8933	{
8934	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8935	Assert(!fBlockSti);
8936	Assert(!fBlockMovSS);
8937	Assert(!fBlockNmi);
8938	}
8939	#endif
8940	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8941	uIntType));
8942
8943	/*
8944	* Inject the event and get any changes to the guest-interruptibility state.
8945	*
8946	* The guest-interruptibility state may need to be updated if we inject the event
8947	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
8948	*/
8949	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
8950	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8951
8952	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8953	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8954	else
8955	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8956	}
8957
8958	/*
8959	* Update the guest-interruptibility state.
8960	*
8961	* This is required for the real-on-v86 software interrupt injection case above, as well as
8962	* updates to the guest state from ring-3 or IEM/REM.
8963	*/
8964	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
8965	AssertRCReturn(rc, rc);
8966
8967	/*
8968	* There's no need to clear the VM-entry interruption-information field here if we're not
8969	* injecting anything. VT-x clears the valid bit on every VM-exit.
8970	*
8971	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8972	*/
8973
8974	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8975	NOREF(fBlockMovSS); NOREF(fBlockSti);
8976	return rcStrict;
8977	}
8978
8979
8980	/**
8981	* Enters the VT-x session.
8982	*
8983	* @returns VBox status code.
8984	* @param pVCpu The cross context virtual CPU structure.
8985	*/
8986	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
8987	{
8988	AssertPtr(pVCpu);
8989	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8990	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8991
8992	LogFlowFunc(("pVCpu=%p\n", pVCpu));
8993	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8994	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8995
8996	#ifdef VBOX_STRICT
8997	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8998	RTCCUINTREG uHostCR4 = ASMGetCR4();
8999	if (!(uHostCR4 & X86_CR4_VMXE))
9000	{
9001	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
9002	return VERR_VMX_X86_CR4_VMXE_CLEARED;
9003	}
9004	#endif
9005
9006	/*
9007	* Load the appropriate VMCS as the current and active one.
9008	*/
9009	PVMXVMCSINFO pVmcsInfo;
9010	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
9011	if (!fInNestedGuestMode)
9012	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
9013	else
9014	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
9015	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
9016	if (RT_SUCCESS(rc))
9017	{
9018	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
9019	pVCpu->hm.s.fLeaveDone = false;
9020	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9021
9022	/*
9023	* Do the EMT scheduled L1D flush here if needed.
9024	*/
9025	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9026	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9027	else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
9028	hmR0MdsClear();
9029	}
9030	return rc;
9031	}
9032
9033
9034	/**
9035	* The thread-context callback (only on platforms which support it).
9036	*
9037	* @param enmEvent The thread-context event.
9038	* @param pVCpu The cross context virtual CPU structure.
9039	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
9040	* @thread EMT(pVCpu)
9041	*/
9042	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
9043	{
9044	NOREF(fGlobalInit);
9045
9046	switch (enmEvent)
9047	{
9048	case RTTHREADCTXEVENT_OUT:
9049	{
9050	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9051	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9052	VMCPU_ASSERT_EMT(pVCpu);
9053
9054	/* No longjmps (logger flushes, locks) in this fragile context. */
9055	VMMRZCallRing3Disable(pVCpu);
9056	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
9057
9058	/* Restore host-state (FPU, debug etc.) */
9059	if (!pVCpu->hm.s.fLeaveDone)
9060	{
9061	/*
9062	* Do -not- import the guest-state here as we might already be in the middle of importing
9063	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
9064	*/
9065	hmR0VmxLeave(pVCpu, false /* fImportState */);
9066	pVCpu->hm.s.fLeaveDone = true;
9067	}
9068
9069	/* Leave HM context, takes care of local init (term). */
9070	int rc = HMR0LeaveCpu(pVCpu);
9071	AssertRC(rc);
9072
9073	/* Restore longjmp state. */
9074	VMMRZCallRing3Enable(pVCpu);
9075	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
9076	break;
9077	}
9078
9079	case RTTHREADCTXEVENT_IN:
9080	{
9081	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9082	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9083	VMCPU_ASSERT_EMT(pVCpu);
9084
9085	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
9086	VMMRZCallRing3Disable(pVCpu);
9087	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
9088
9089	/* Initialize the bare minimum state required for HM. This takes care of
9090	initializing VT-x if necessary (onlined CPUs, local init etc.) */
9091	int rc = hmR0EnterCpu(pVCpu);
9092	AssertRC(rc);
9093	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9094	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9095
9096	/* Load the active VMCS as the current one. */
9097	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9098	rc = hmR0VmxLoadVmcs(pVmcsInfo);
9099	AssertRC(rc);
9100	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9101	pVCpu->hm.s.fLeaveDone = false;
9102
9103	/* Do the EMT scheduled L1D flush if needed. */
9104	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9105	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9106
9107	/* Restore longjmp state. */
9108	VMMRZCallRing3Enable(pVCpu);
9109	break;
9110	}
9111
9112	default:
9113	break;
9114	}
9115	}
9116
9117
9118	/**
9119	* Exports the host state into the VMCS host-state area.
9120	* Sets up the VM-exit MSR-load area.
9121	*
9122	* The CPU state will be loaded from these fields on every successful VM-exit.
9123	*
9124	* @returns VBox status code.
9125	* @param pVCpu The cross context virtual CPU structure.
9126	*
9127	* @remarks No-long-jump zone!!!
9128	*/
9129	static int hmR0VmxExportHostState(PVMCPU pVCpu)
9130	{
9131	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9132
9133	int rc = VINF_SUCCESS;
9134	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
9135	{
9136	rc = hmR0VmxExportHostControlRegs();
9137	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9138
9139	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
9140	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9141
9142	rc = hmR0VmxExportHostMsrs(pVCpu);
9143	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9144
9145	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
9146	}
9147	return rc;
9148	}
9149
9150
9151	/**
9152	* Saves the host state in the VMCS host-state.
9153	*
9154	* @returns VBox status code.
9155	* @param pVCpu The cross context virtual CPU structure.
9156	*
9157	* @remarks No-long-jump zone!!!
9158	*/
9159	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
9160	{
9161	AssertPtr(pVCpu);
9162	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9163
9164	/*
9165	* Export the host state here while entering HM context.
9166	* When thread-context hooks are used, we might get preempted and have to re-save the host
9167	* state but most of the time we won't be, so do it here before we disable interrupts.
9168	*/
9169	return hmR0VmxExportHostState(pVCpu);
9170	}
9171
9172
9173	/**
9174	* Exports the guest state into the VMCS guest-state area.
9175	*
9176	* The will typically be done before VM-entry when the guest-CPU state and the
9177	* VMCS state may potentially be out of sync.
9178	*
9179	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9180	* VM-entry controls.
9181	* Sets up the appropriate VMX non-root function to execute guest code based on
9182	* the guest CPU mode.
9183	*
9184	* @returns VBox strict status code.
9185	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9186	* without unrestricted guest execution and the VMMDev is not presently
9187	* mapped (e.g. EFI32).
9188	*
9189	* @param pVCpu The cross context virtual CPU structure.
9190	* @param pVmxTransient The VMX-transient structure.
9191	*
9192	* @remarks No-long-jump zone!!!
9193	*/
9194	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9195	{
9196	AssertPtr(pVCpu);
9197	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9198	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9199
9200	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9201
9202	/*
9203	* Determine real-on-v86 mode.
9204	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9205	*/
9206	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9207	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9208	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9209	pVmcsInfo->RealMode. fRealOnV86Active = false;
9210	else
9211	{
9212	Assert(!pVmxTransient->fIsNestedGuest);
9213	pVmcsInfo->RealMode.fRealOnV86Active = true;
9214	}
9215
9216	/*
9217	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9218	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9219	*/
9220	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9221	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9222	* be a need to evaluate this everytime since I'm pretty sure we intercept
9223	* all guest paging mode changes. */
9224	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9225	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9226
9227	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9228	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9229
9230	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9231	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9232
9233	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9234	if (rcStrict == VINF_SUCCESS)
9235	{ /* likely */ }
9236	else
9237	{
9238	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9239	return rcStrict;
9240	}
9241
9242	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9243	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9244
9245	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9246	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9247
9248	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9249	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9250
9251	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9252	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9253
9254	rc = hmR0VmxExportGuestRip(pVCpu);
9255	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9256	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9257	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9258
9259	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9260	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9261	\| HM_CHANGED_GUEST_CR2
9262	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9263	\| HM_CHANGED_GUEST_X87
9264	\| HM_CHANGED_GUEST_SSE_AVX
9265	\| HM_CHANGED_GUEST_OTHER_XSAVE
9266	\| HM_CHANGED_GUEST_XCRx
9267	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9268	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9269	\| HM_CHANGED_GUEST_TSC_AUX
9270	\| HM_CHANGED_GUEST_OTHER_MSRS
9271	\| HM_CHANGED_GUEST_HWVIRT /* More accurate PLE handling someday? */
9272	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9273
9274	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9275	return rc;
9276	}
9277
9278
9279	/**
9280	* Exports the state shared between the host and guest into the VMCS.
9281	*
9282	* @param pVCpu The cross context virtual CPU structure.
9283	* @param pVmxTransient The VMX-transient structure.
9284	*
9285	* @remarks No-long-jump zone!!!
9286	*/
9287	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9288	{
9289	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9290	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9291
9292	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9293	{
9294	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9295	AssertRC(rc);
9296	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9297
9298	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9299	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9300	{
9301	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9302	AssertRC(rc);
9303	}
9304	}
9305
9306	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9307	{
9308	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9309	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9310	}
9311
9312	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9313	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9314	}
9315
9316
9317	/**
9318	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9319	*
9320	* @returns Strict VBox status code (i.e. informational status codes too).
9321	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9322	* without unrestricted guest execution and the VMMDev is not presently
9323	* mapped (e.g. EFI32).
9324	*
9325	* @param pVCpu The cross context virtual CPU structure.
9326	* @param pVmxTransient The VMX-transient structure.
9327	*
9328	* @remarks No-long-jump zone!!!
9329	*/
9330	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9331	{
9332	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9333	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9334	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9335
9336	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9337	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9338	#endif
9339
9340	/*
9341	* For many exits it's only RIP that changes and hence try to export it first
9342	* without going through a lot of change flag checks.
9343	*/
9344	VBOXSTRICTRC rcStrict;
9345	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9346	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9347	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9348	{
9349	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9350	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9351	{ /* likely */}
9352	else
9353	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9354	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9355	}
9356	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9357	{
9358	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9359	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9360	{ /* likely */}
9361	else
9362	{
9363	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9364	VBOXSTRICTRC_VAL(rcStrict)));
9365	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9366	return rcStrict;
9367	}
9368	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9369	}
9370	else
9371	rcStrict = VINF_SUCCESS;
9372
9373	#ifdef VBOX_STRICT
9374	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9375	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9376	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9377	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9378	("fCtxChanged=%#RX64\n", fCtxChanged));
9379	#endif
9380	return rcStrict;
9381	}
9382
9383
9384	/**
9385	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9386	* and update error record fields accordingly.
9387	*
9388	* @return VMX_IGS_* return codes.
9389	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9390	* wrong with the guest state.
9391	*
9392	* @param pVCpu The cross context virtual CPU structure.
9393	* @param pVmcsInfo The VMCS info. object.
9394	*
9395	* @remarks This function assumes our cache of the VMCS controls
9396	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9397	*/
9398	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
9399	{
9400	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9401	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9402	uError = (err); \
9403	break; \
9404	} else do { } while (0)
9405
9406	int rc;
9407	PVM pVM = pVCpu->CTX_SUFF(pVM);
9408	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9409	uint32_t uError = VMX_IGS_ERROR;
9410	uint32_t u32Val;
9411	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9412
9413	do
9414	{
9415	/*
9416	* CR0.
9417	*/
9418	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9419	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9420	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
9421	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9422	if (fUnrestrictedGuest)
9423	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
9424
9425	uint32_t u32GuestCr0;
9426	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
9427	AssertRCBreak(rc);
9428	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9429	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9430	if ( !fUnrestrictedGuest
9431	&& (u32GuestCr0 & X86_CR0_PG)
9432	&& !(u32GuestCr0 & X86_CR0_PE))
9433	{
9434	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9435	}
9436
9437	/*
9438	* CR4.
9439	*/
9440	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9441	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9442
9443	uint32_t u32GuestCr4;
9444	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
9445	AssertRCBreak(rc);
9446	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9447	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9448
9449	/*
9450	* IA32_DEBUGCTL MSR.
9451	*/
9452	uint64_t u64Val;
9453	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9454	AssertRCBreak(rc);
9455	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9456	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9457	{
9458	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9459	}
9460	uint64_t u64DebugCtlMsr = u64Val;
9461
9462	#ifdef VBOX_STRICT
9463	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9464	AssertRCBreak(rc);
9465	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9466	#endif
9467	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9468
9469	/*
9470	* RIP and RFLAGS.
9471	*/
9472	uint32_t u32Eflags;
9473	#if HC_ARCH_BITS == 64
9474	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9475	AssertRCBreak(rc);
9476	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9477	if ( !fLongModeGuest
9478	\|\| !pCtx->cs.Attr.n.u1Long)
9479	{
9480	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9481	}
9482	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9483	* must be identical if the "IA-32e mode guest" VM-entry
9484	* control is 1 and CS.L is 1. No check applies if the
9485	* CPU supports 64 linear-address bits. */
9486
9487	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9488	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9489	AssertRCBreak(rc);
9490	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9491	VMX_IGS_RFLAGS_RESERVED);
9492	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9493	u32Eflags = u64Val;
9494	#else
9495	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9496	AssertRCBreak(rc);
9497	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9498	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9499	#endif
9500
9501	if ( fLongModeGuest
9502	\|\| ( fUnrestrictedGuest
9503	&& !(u32GuestCr0 & X86_CR0_PE)))
9504	{
9505	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9506	}
9507
9508	uint32_t u32EntryInfo;
9509	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9510	AssertRCBreak(rc);
9511	if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9512	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9513	{
9514	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9515	}
9516
9517	/*
9518	* 64-bit checks.
9519	*/
9520	#if HC_ARCH_BITS == 64
9521	if (fLongModeGuest)
9522	{
9523	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9524	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9525	}
9526
9527	if ( !fLongModeGuest
9528	&& (u32GuestCr4 & X86_CR4_PCIDE))
9529	{
9530	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9531	}
9532
9533	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9534	* 51:32 beyond the processor's physical-address width are 0. */
9535
9536	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9537	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9538	{
9539	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9540	}
9541
9542	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9543	AssertRCBreak(rc);
9544	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9545
9546	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9547	AssertRCBreak(rc);
9548	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9549	#endif
9550
9551	/*
9552	* PERF_GLOBAL MSR.
9553	*/
9554	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9555	{
9556	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9557	AssertRCBreak(rc);
9558	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9559	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9560	}
9561
9562	/*
9563	* PAT MSR.
9564	*/
9565	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9566	{
9567	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9568	AssertRCBreak(rc);
9569	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9570	for (unsigned i = 0; i < 8; i++)
9571	{
9572	uint8_t u8Val = (u64Val & 0xff);
9573	if ( u8Val != 0 /* UC */
9574	&& u8Val != 1 /* WC */
9575	&& u8Val != 4 /* WT */
9576	&& u8Val != 5 /* WP */
9577	&& u8Val != 6 /* WB */
9578	&& u8Val != 7 /* UC- */)
9579	{
9580	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9581	}
9582	u64Val >>= 8;
9583	}
9584	}
9585
9586	/*
9587	* EFER MSR.
9588	*/
9589	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9590	{
9591	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9592	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9593	AssertRCBreak(rc);
9594	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9595	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9596	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9597	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9598	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9599	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9600	* iemVmxVmentryCheckGuestState(). */
9601	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9602	\|\| !(u32GuestCr0 & X86_CR0_PG)
9603	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9604	VMX_IGS_EFER_LMA_LME_MISMATCH);
9605	}
9606
9607	/*
9608	* Segment registers.
9609	*/
9610	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9611	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9612	if (!(u32Eflags & X86_EFL_VM))
9613	{
9614	/* CS */
9615	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9616	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9617	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9618	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9619	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9620	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9621	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9622	/* CS cannot be loaded with NULL in protected mode. */
9623	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9624	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9625	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9626	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9627	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9628	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9629	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9630	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9631	else
9632	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9633
9634	/* SS */
9635	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9636	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9637	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9638	if ( !(pCtx->cr0 & X86_CR0_PE)
9639	\|\| pCtx->cs.Attr.n.u4Type == 3)
9640	{
9641	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9642	}
9643	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9644	{
9645	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9646	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9647	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9648	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9649	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9650	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9651	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9652	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9653	}
9654
9655	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9656	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9657	{
9658	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9659	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9660	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9661	\|\| pCtx->ds.Attr.n.u4Type > 11
9662	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9663	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9664	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9665	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9666	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9667	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9668	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9669	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9670	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9671	}
9672	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9673	{
9674	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9675	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9676	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9677	\|\| pCtx->es.Attr.n.u4Type > 11
9678	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9679	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9680	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9681	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9682	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9683	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9684	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9685	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9686	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9687	}
9688	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9689	{
9690	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9691	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9692	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9693	\|\| pCtx->fs.Attr.n.u4Type > 11
9694	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9695	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9696	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9697	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9698	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9699	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9700	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9701	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9702	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9703	}
9704	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9705	{
9706	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9707	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9708	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9709	\|\| pCtx->gs.Attr.n.u4Type > 11
9710	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9711	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9712	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9713	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9714	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9715	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9716	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9717	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9718	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9719	}
9720	/* 64-bit capable CPUs. */
9721	#if HC_ARCH_BITS == 64
9722	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9723	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9724	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9725	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9726	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9727	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9728	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9729	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9730	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9731	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9732	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9733	#endif
9734	}
9735	else
9736	{
9737	/* V86 mode checks. */
9738	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9739	if (pVmcsInfo->RealMode.fRealOnV86Active)
9740	{
9741	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9742	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9743	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9744	}
9745	else
9746	{
9747	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9748	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9749	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9750	}
9751
9752	/* CS */
9753	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9754	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9755	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9756	/* SS */
9757	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9758	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9759	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9760	/* DS */
9761	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9762	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9763	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9764	/* ES */
9765	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9766	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9767	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9768	/* FS */
9769	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9770	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9771	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9772	/* GS */
9773	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9774	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9775	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9776	/* 64-bit capable CPUs. */
9777	#if HC_ARCH_BITS == 64
9778	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9779	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9780	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9781	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9782	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9783	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9784	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9785	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9786	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9787	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9788	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9789	#endif
9790	}
9791
9792	/*
9793	* TR.
9794	*/
9795	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9796	/* 64-bit capable CPUs. */
9797	#if HC_ARCH_BITS == 64
9798	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9799	#endif
9800	if (fLongModeGuest)
9801	{
9802	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9803	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9804	}
9805	else
9806	{
9807	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9808	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9809	VMX_IGS_TR_ATTR_TYPE_INVALID);
9810	}
9811	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9812	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9813	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9814	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9815	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9816	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9817	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9818	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9819
9820	/*
9821	* GDTR and IDTR.
9822	*/
9823	#if HC_ARCH_BITS == 64
9824	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9825	AssertRCBreak(rc);
9826	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9827
9828	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9829	AssertRCBreak(rc);
9830	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9831	#endif
9832
9833	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9834	AssertRCBreak(rc);
9835	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9836
9837	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9838	AssertRCBreak(rc);
9839	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9840
9841	/*
9842	* Guest Non-Register State.
9843	*/
9844	/* Activity State. */
9845	uint32_t u32ActivityState;
9846	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9847	AssertRCBreak(rc);
9848	HMVMX_CHECK_BREAK( !u32ActivityState
9849	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9850	VMX_IGS_ACTIVITY_STATE_INVALID);
9851	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9852	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9853	uint32_t u32IntrState;
9854	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9855	AssertRCBreak(rc);
9856	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9857	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9858	{
9859	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9860	}
9861
9862	/** @todo Activity state and injecting interrupts. Left as a todo since we
9863	* currently don't use activity states but ACTIVE. */
9864
9865	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9866	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9867
9868	/* Guest interruptibility-state. */
9869	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9870	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9871	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9872	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9873	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9874	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9875	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9876	if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
9877	{
9878	if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9879	{
9880	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9881	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9882	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9883	}
9884	else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9885	{
9886	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9887	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9888	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9889	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9890	}
9891	}
9892	/** @todo Assumes the processor is not in SMM. */
9893	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9894	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9895	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9896	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9897	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9898	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9899	&& VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9900	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9901	{
9902	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
9903	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9904	}
9905
9906	/* Pending debug exceptions. */
9907	#if HC_ARCH_BITS == 64
9908	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
9909	AssertRCBreak(rc);
9910	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9911	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9912	u32Val = u64Val; /* For pending debug exceptions checks below. */
9913	#else
9914	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
9915	AssertRCBreak(rc);
9916	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9917	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9918	#endif
9919
9920	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9921	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
9922	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9923	{
9924	if ( (u32Eflags & X86_EFL_TF)
9925	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9926	{
9927	/* Bit 14 is PendingDebug.BS. */
9928	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9929	}
9930	if ( !(u32Eflags & X86_EFL_TF)
9931	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9932	{
9933	/* Bit 14 is PendingDebug.BS. */
9934	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9935	}
9936	}
9937
9938	/* VMCS link pointer. */
9939	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9940	AssertRCBreak(rc);
9941	if (u64Val != UINT64_C(0xffffffffffffffff))
9942	{
9943	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9944	/** @todo Bits beyond the processor's physical-address width MBZ. */
9945	/** @todo 32-bit located in memory referenced by value of this field (as a
9946	* physical address) must contain the processor's VMCS revision ID. */
9947	/** @todo SMM checks. */
9948	}
9949
9950	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9951	* not using nested paging? */
9952	if ( pVM->hm.s.fNestedPaging
9953	&& !fLongModeGuest
9954	&& CPUMIsGuestInPAEModeEx(pCtx))
9955	{
9956	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9957	AssertRCBreak(rc);
9958	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9959
9960	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9961	AssertRCBreak(rc);
9962	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9963
9964	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9965	AssertRCBreak(rc);
9966	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9967
9968	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9969	AssertRCBreak(rc);
9970	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9971	}
9972
9973	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9974	if (uError == VMX_IGS_ERROR)
9975	uError = VMX_IGS_REASON_NOT_FOUND;
9976	} while (0);
9977
9978	pVCpu->hm.s.u32HMError = uError;
9979	return uError;
9980
9981	#undef HMVMX_ERROR_BREAK
9982	#undef HMVMX_CHECK_BREAK
9983	}
9984
9985
9986	/**
9987	* Setup the APIC-access page for virtualizing APIC access.
9988	*
9989	* This can cause a longjumps to R3 due to the acquisition of the PGM lock, hence
9990	* this not done as part of exporting guest state, see @bugref{8721}.
9991	*
9992	* @returns VBox status code.
9993	* @param pVCpu The cross context virtual CPU structure.
9994	*/
9995	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
9996	{
9997	PVM pVM = pVCpu->CTX_SUFF(pVM);
9998	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
9999
10000	Assert(PDMHasApic(pVM));
10001	Assert(u64MsrApicBase);
10002
10003	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
10004	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
10005
10006	/* Unalias any existing mapping. */
10007	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
10008	AssertRCReturn(rc, rc);
10009
10010	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
10011	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
10012	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
10013	AssertRCReturn(rc, rc);
10014
10015	/* Update the per-VCPU cache of the APIC base MSR. */
10016	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
10017	return VINF_SUCCESS;
10018	}
10019
10020
10021	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10022	/**
10023	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
10024	* nested-guest using hardware-assisted VMX.
10025	*
10026	* @param pVCpu The cross context virtual CPU structure.
10027	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
10028	* @param pVmcsInfoGst The guest VMCS info. object.
10029	*/
10030	static void hmR0VmxMergeMsrBitmapNested(PCVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
10031	{
10032	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
10033	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
10034	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
10035	Assert(pu64MsrBitmapNstGst);
10036	Assert(pu64MsrBitmapGst);
10037	Assert(pu64MsrBitmap);
10038
10039	/*
10040	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
10041	* MSR that is intercepted by the guest is also intercepted while executing the
10042	* nested-guest using hardware-assisted VMX.
10043	*/
10044	uint32_t const cbFrag = sizeof(uint64_t);
10045	uint32_t const cFrags = X86_PAGE_4K_SIZE / cbFrag;
10046	for (uint32_t i = 0; i <= cFrags; i++)
10047	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
10048	}
10049
10050
10051	/**
10052	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
10053	* hardware-assisted VMX execution of the nested-guest.
10054	*
10055	* For a guest, we don't modify these controls once we set up the VMCS and hence
10056	* this function is never called.
10057	*
10058	* For nested-guests since the guest hypervisor provides these controls on every
10059	* nested-guest VM-entry and could potentially change them everytime we need to
10060	* merge them before every nested-guest VM-entry.
10061	*
10062	* @returns VBox status code.
10063	* @param pVCpu The cross context virtual CPU structure.
10064	*/
10065	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
10066	{
10067	PVM pVM = pVCpu->CTX_SUFF(pVM);
10068	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
10069	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10070	Assert(pVmcsNstGst);
10071
10072	/*
10073	* Merge the controls with the requirements of the guest VMCS.
10074	*
10075	* We do not need to validate the nested-guest VMX features specified in the
10076	* nested-guest VMCS with the features supported by the physical CPU as it's
10077	* already done by the VMLAUNCH/VMRESUME instruction emulation.
10078	*
10079	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the
10080	* guest are derived from the VMX features supported by the physical CPU.
10081	*/
10082
10083	/* Pin-based VM-execution controls. */
10084	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
10085
10086	/* Processor-based VM-execution controls. */
10087	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
10088	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
10089	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
10090	\| VMX_PROC_CTLS_USE_TPR_SHADOW
10091	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
10092
10093	/* Secondary processor-based VM-execution controls. */
10094	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
10095	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
10096	\| VMX_PROC_CTLS2_INVPCID
10097	\| VMX_PROC_CTLS2_RDTSCP
10098	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
10099	\| VMX_PROC_CTLS2_APIC_REG_VIRT
10100	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
10101	\| VMX_PROC_CTLS2_VMFUNC));
10102
10103	/*
10104	* VM-entry controls:
10105	* These controls contains state that depends on the nested-guest state (primarily
10106	* EFER MSR) and is thus not constant through VMLAUNCH/VMRESUME and the nested-guest
10107	* VM-exit. Although the nested-hypervisor cannot change it, we need to in order to
10108	* properly continue executing the nested-guest if the EFER MSR changes but does not
10109	* cause a nested-guest VM-exits.
10110	*
10111	* VM-exit controls:
10112	* These controls specify the host state on return. We cannot use the controls from
10113	* the nested-hypervisor state as is as it would contain the guest state rather than
10114	* the host state. Since the host state is subject to change (e.g. preemption, trips
10115	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
10116	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
10117	*
10118	* VM-entry MSR-load:
10119	* The guest MSRs from the VM-entry MSR-load area are already loaded into the
10120	* guest-CPU context by the VMLAUNCH/VMRESUME instruction emulation.
10121	*
10122	* VM-exit MSR-store:
10123	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU
10124	* context back into the VM-exit MSR-store area.
10125	*
10126	* VM-exit MSR-load areas:
10127	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence,
10128	* we can entirely ignore what the nested-hypervisor wants to load here.
10129	*/
10130
10131	/*
10132	* Exception bitmap.
10133	*
10134	* We could remove #UD from the guest bitmap and merge it with the nested-guest
10135	* bitmap here (and avoid doing anything while exporting nested-guest state), but to
10136	* keep the code more flexible if intercepting exceptions become more dynamic in
10137	* the future we do it as part of exporting the nested-guest state.
10138	*/
10139	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
10140
10141	/*
10142	* CR0/CR4 guest/host mask.
10143	*
10144	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest
10145	* must cause VM-exits, so we need to merge them here.
10146	*/
10147	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
10148	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
10149
10150	/*
10151	* Page-fault error-code mask and match.
10152	*
10153	* Although we require unrestricted guest execution (and thereby nested-paging) for
10154	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
10155	* normally intercept #PFs, it might intercept them for debugging purposes.
10156	*
10157	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF
10158	* filters. If the outer guest is intercepting #PFs we must intercept all #PFs.
10159	*/
10160	uint32_t u32XcptPFMask;
10161	uint32_t u32XcptPFMatch;
10162	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10163	{
10164	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10165	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10166	}
10167	else
10168	{
10169	u32XcptPFMask = 0;
10170	u32XcptPFMatch = 0;
10171	}
10172
10173	/*
10174	* Pause-Loop exiting.
10175	*/
10176	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10177	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10178
10179	/*
10180	* I/O Bitmap.
10181	*
10182	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we
10183	* always intercept all I/O port accesses.
10184	*/
10185	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10186
10187	/*
10188	* APIC-access page.
10189	*
10190	* The APIC-access page address has already been initialized while setting up the
10191	* nested-guest VMCS. In theory, even if the guest-physical address is invalid, it
10192	* should not be on any consequence to the host or to the guest for that matter, but
10193	* we only accept valid addresses verified by the VMLAUNCH/VMRESUME instruction
10194	* emulation to keep it simple.
10195	*/
10196
10197	/*
10198	* Virtual-APIC page and TPR threshold.
10199	*
10200	* We shall use the host-physical address of the virtual-APIC page in guest memory directly.
10201	* For this reason, we can access the virtual-APIC page of the nested-guest only using
10202	* PGM physical handlers as we must not assume a kernel virtual-address mapping exists and
10203	* requesting PGM for a mapping could be expensive/resource intensive (PGM mapping cache).
10204	*/
10205	RTHCPHYS HCPhysVirtApic = NIL_RTHCPHYS;
10206	uint32_t const u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10207	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10208	{
10209	int rc = PGMPhysGCPhys2HCPhys(pVM, pVmcsNstGst->u64AddrVirtApic.u, &HCPhysVirtApic);
10210
10211	/*
10212	* If the guest hypervisor has loaded crap into the virtual-APIC page field
10213	* we would fail to obtain a valid host-physical address for its guest-physical
10214	* address.
10215	*
10216	* We currently do not support this scenario. Maybe in the future if there is a
10217	* pressing need we can explore making this particular set of conditions work.
10218	* Right now we just cause a VM-entry failure.
10219	*
10220	* This has already been checked by VMLAUNCH/VMRESUME instruction emulation,
10221	* so should not really failure at the moment.
10222	*/
10223	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10224	}
10225	else
10226	{
10227	/*
10228	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10229	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10230	* be taken care of by EPT/shadow paging.
10231	*/
10232	if (pVM->hm.s.fAllow64BitGuests)
10233	{
10234	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10235	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10236	}
10237	}
10238
10239	/*
10240	* Validate basic assumptions.
10241	*/
10242	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10243	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10244	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10245	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10246
10247	/*
10248	* Commit it to the nested-guest VMCS.
10249	*/
10250	int rc = VINF_SUCCESS;
10251	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10252	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10253	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10254	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10255	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10256	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10257	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10258	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10259	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10260	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10261	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10262	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10263	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10264	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10265	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10266	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10267	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10268	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10269	{
10270	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10271	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10272	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10273	}
10274	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10275	{
10276	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10277	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10278	}
10279	AssertRCReturn(rc, rc);
10280
10281	/*
10282	* Update the nested-guest VMCS cache.
10283	*/
10284	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10285	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10286	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10287	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10288	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10289	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10290	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10291	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10292	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10293
10294	/*
10295	* MSR bitmap.
10296	*
10297	* The MSR bitmap address has already been initialized while setting up the
10298	* nested-guest VMCS, here we need to merge the MSR bitmaps.
10299	*/
10300	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10301	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10302
10303	return VINF_SUCCESS;
10304	}
10305	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10306
10307
10308	/**
10309	* Does the preparations before executing guest code in VT-x.
10310	*
10311	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10312	* recompiler/IEM. We must be cautious what we do here regarding committing
10313	* guest-state information into the VMCS assuming we assuredly execute the
10314	* guest in VT-x mode.
10315	*
10316	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10317	* the common-state (TRPM/forceflags), we must undo those changes so that the
10318	* recompiler/IEM can (and should) use them when it resumes guest execution.
10319	* Otherwise such operations must be done when we can no longer exit to ring-3.
10320	*
10321	* @returns Strict VBox status code (i.e. informational status codes too).
10322	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10323	* have been disabled.
10324	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10325	* double-fault into the guest.
10326	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10327	* dispatched directly.
10328	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10329	*
10330	* @param pVCpu The cross context virtual CPU structure.
10331	* @param pVmxTransient The VMX-transient structure.
10332	* @param fStepping Whether we are single-stepping the guest in the
10333	* hypervisor debugger. Makes us ignore some of the reasons
10334	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10335	* if event dispatching took place.
10336	*/
10337	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10338	{
10339	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10340
10341	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10342	if (pVmxTransient->fIsNestedGuest)
10343	{
10344	RT_NOREF2(pVCpu, fStepping);
10345	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10346	return VINF_EM_RESCHEDULE_REM;
10347	}
10348	#endif
10349
10350	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10351	PGMRZDynMapFlushAutoSet(pVCpu);
10352	#endif
10353
10354	/*
10355	* Check and process force flag actions, some of which might require us to go back to ring-3.
10356	*/
10357	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10358	if (rcStrict == VINF_SUCCESS)
10359	{ /* FFs don't get set all the time. */ }
10360	else
10361	return rcStrict;
10362
10363	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10364	/*
10365	* Switch to the nested-guest VMCS as we may have transitioned into executing
10366	* the nested-guest without leaving ring-0. Otherwise, if we came from ring-3
10367	* we would load the nested-guest VMCS while entering the VMX ring-0 session.
10368	*
10369	* We do this as late as possible to minimize (though not completely remove)
10370	* clearing/loading VMCS again due to premature trips to ring-3 above.
10371	*/
10372	if (pVmxTransient->fIsNestedGuest)
10373	{
10374	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10375	{
10376	/*
10377	* Ensure we have synced everything from the guest VMCS and also flag that
10378	* that we need to export the full (nested) guest-CPU context to the
10379	* nested-guest VMCS.
10380	*/
10381	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
10382	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST);
10383
10384	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
10385	int rc = hmR0VmxSwitchVmcs(&pVCpu->hm.s.vmx.VmcsInfo, &pVCpu->hm.s.vmx.VmcsInfoNstGst);
10386	if (RT_LIKELY(rc == VINF_SUCCESS))
10387	{
10388	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = true;
10389	ASMSetFlags(fEFlags);
10390	pVmxTransient->pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10391
10392	/*
10393	* We use a different VM-exit MSR-store area for the nested-guest. Hence,
10394	* flag that we need to update the host MSR values there. Even if we decide
10395	* in the future to share the VM-exit MSR-store area page with the guest,
10396	* if its content differs, we would have to update the host MSRs anyway.
10397	*/
10398	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
10399	Assert(!pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer); /** @todo NSTVMX: Paranoia remove later. */
10400	}
10401	else
10402	{
10403	ASMSetFlags(fEFlags);
10404	return rc;
10405	}
10406	}
10407
10408	/*
10409	* Merge guest VMCS controls with the nested-guest VMCS controls.
10410	*
10411	* Even if we have not executed the guest prior to this (e.g. when resuming
10412	* from a saved state), we should be okay with merging controls as we
10413	* initialize the guest VMCS controls as part of VM setup phase.
10414	*/
10415	if (!pVCpu->hm.s.vmx.fMergedNstGstCtls)
10416	{
10417	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10418	AssertRCReturn(rc, rc);
10419	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10420	}
10421	}
10422	#endif
10423
10424	/*
10425	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10426	* We look at the guest VMCS control here as we always set it when supported by
10427	* the physical CPU. Looking at the nested-guest control here would not be
10428	* possible because they are not merged yet.
10429	*/
10430	PVM pVM = pVCpu->CTX_SUFF(pVM);
10431	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10432	Assert(pVmcsInfo);
10433	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10434	&& (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10435	&& PDMHasApic(pVM))
10436	{
10437	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10438	AssertRCReturn(rc, rc);
10439	}
10440
10441	/*
10442	* Evaluate events to be injected into the guest.
10443	*
10444	* Events in TRPM can be injected without inspecting the guest state.
10445	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10446	* guest to cause a VM-exit the next time they are ready to receive the event.
10447	*/
10448	if (TRPMHasTrap(pVCpu))
10449	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10450
10451	uint32_t fIntrState;
10452	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10453
10454	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10455	/*
10456	* While evaluating pending events if something failed (unlikely) or if we were
10457	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10458	*/
10459	if ( rcStrict != VINF_SUCCESS
10460	\|\| ( pVmxTransient->fIsNestedGuest
10461	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)))
10462	return rcStrict;
10463	#endif
10464
10465	/*
10466	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10467	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10468	* also result in triple-faulting the VM.
10469	*
10470	* The above does not apply when executing a nested-guest (since unrestricted guest execution
10471	* is a requirement) regardless doing it avoid duplicating code elsewhere.
10472	*/
10473	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10474	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10475	{ /* likely */ }
10476	else
10477	{
10478	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10479	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10480	return rcStrict;
10481	}
10482
10483	/*
10484	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10485	* import CR3 themselves. We will need to update them here, as even as late as the above
10486	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10487	* the below force flags to be set.
10488	*/
10489	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10490	{
10491	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10492	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10493	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10494	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10495	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10496	}
10497	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10498	{
10499	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10500	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10501	}
10502
10503	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10504	/* Paranoia. */
10505	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10506	#endif
10507
10508	/*
10509	* No longjmps to ring-3 from this point on!!!
10510	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10511	* This also disables flushing of the R0-logger instance (if any).
10512	*/
10513	VMMRZCallRing3Disable(pVCpu);
10514
10515	/*
10516	* Export the guest state bits.
10517	*
10518	* We cannot perform longjmps while loading the guest state because we do not preserve the
10519	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10520	* CPU migration.
10521	*
10522	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10523	* registers. Hence, loading of the guest state needs to be done -after- injection of events.
10524	*/
10525	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10526	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10527	{ /* likely */ }
10528	else
10529	{
10530	VMMRZCallRing3Enable(pVCpu);
10531	return rcStrict;
10532	}
10533
10534	/*
10535	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10536	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10537	* preemption disabled for a while. Since this is purely to aid the
10538	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10539	* disable interrupt on NT.
10540	*
10541	* We need to check for force-flags that could've possible been altered since we last
10542	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10543	* see @bugref{6398}).
10544	*
10545	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10546	* to ring-3 before executing guest code.
10547	*/
10548	pVmxTransient->fEFlags = ASMIntDisableFlags();
10549
10550	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10551	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10552	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10553	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10554	{
10555	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10556	{
10557	pVCpu->hm.s.Event.fPending = false;
10558
10559	/*
10560	* We've injected any pending events. This is really the point of no return (to ring-3).
10561	*
10562	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10563	* returns from this function, so don't enable them here.
10564	*/
10565	return VINF_SUCCESS;
10566	}
10567
10568	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10569	rcStrict = VINF_EM_RAW_INTERRUPT;
10570	}
10571	else
10572	{
10573	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10574	rcStrict = VINF_EM_RAW_TO_R3;
10575	}
10576
10577	ASMSetFlags(pVmxTransient->fEFlags);
10578	VMMRZCallRing3Enable(pVCpu);
10579
10580	return rcStrict;
10581	}
10582
10583
10584	/**
10585	* Final preparations before executing guest code using hardware-assisted VMX.
10586	*
10587	* We can no longer get preempted to a different host CPU and there are no returns
10588	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10589	* failures), this function is not intended to fail sans unrecoverable hardware
10590	* errors.
10591	*
10592	* @param pVCpu The cross context virtual CPU structure.
10593	* @param pVmxTransient The VMX-transient structure.
10594	*
10595	* @remarks Called with preemption disabled.
10596	* @remarks No-long-jump zone!!!
10597	*/
10598	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10599	{
10600	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10601	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10602	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10603	Assert(!pVCpu->hm.s.Event.fPending);
10604
10605	/*
10606	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10607	*/
10608	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10609	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10610
10611	PVM pVM = pVCpu->CTX_SUFF(pVM);
10612	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10613
10614	if (!CPUMIsGuestFPUStateActive(pVCpu))
10615	{
10616	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10617	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10618	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10619	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10620	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10621	}
10622
10623	/*
10624	* Re-save the host state bits as we may've been preempted (only happens when
10625	* thread-context hooks are used or when the VM start function changes).
10626	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10627	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10628	* see @bugref{8432}.
10629	*
10630	* This may also happen when switching to/from a nested-guest VMCS without leaving
10631	* ring-0.
10632	*/
10633	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10634	{
10635	int rc = hmR0VmxExportHostState(pVCpu);
10636	AssertRC(rc);
10637	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
10638	}
10639	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10640
10641	/*
10642	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10643	*/
10644	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10645	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10646	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10647
10648	/*
10649	* Store status of the shared guest/host debug state at the time of VM-entry.
10650	*/
10651	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
10652	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
10653	{
10654	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
10655	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
10656	}
10657	else
10658	#endif
10659	{
10660	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10661	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10662	}
10663
10664	/*
10665	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10666	* more than one conditional check. The post-run side of our code shall determine
10667	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10668	*/
10669	if (pVmcsInfo->pbVirtApic)
10670	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10671
10672	/*
10673	* Update the host MSRs values in the VM-exit MSR-load area.
10674	*/
10675	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10676	{
10677	if (pVmcsInfo->cExitMsrLoad > 0)
10678	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10679	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10680	}
10681
10682	/*
10683	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10684	* VMX-preemption timer based on the next virtual sync clock deadline.
10685	*/
10686	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10687	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10688	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10689	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10690	{
10691	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10692	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10693	}
10694
10695	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10696	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10697	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10698	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
10699
10700	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10701
10702	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10703	as we're about to start executing the guest . */
10704
10705	/*
10706	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10707	*
10708	* This is done this late as updating the TSC offsetting/preemption timer above
10709	* figures out if we can skip intercepting RDTSCP by calculating the number of
10710	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10711	*/
10712	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10713	{
10714	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10715	{
10716	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10717	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
10718	it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
10719	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10720	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10721	AssertRC(rc);
10722	}
10723	else
10724	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
10725	}
10726
10727	#ifdef VBOX_STRICT
10728	Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
10729	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo);
10730	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10731	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo));
10732	#endif
10733
10734	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10735	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10736	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
10737	* see @bugref{9180#c54}. */
10738	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10739	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10740	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10741	#endif
10742	}
10743
10744
10745	/**
10746	* First C routine invoked after running guest code using hardware-assisted VMX.
10747	*
10748	* @param pVCpu The cross context virtual CPU structure.
10749	* @param pVmxTransient The VMX-transient structure.
10750	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10751	*
10752	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10753	*
10754	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10755	* unconditionally when it is safe to do so.
10756	*/
10757	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10758	{
10759	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10760
10761	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10762	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10763	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10764	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10765	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10766	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10767
10768	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10769	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10770	{
10771	uint64_t uGstTsc;
10772	if (!pVmxTransient->fIsNestedGuest)
10773	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10774	else
10775	{
10776	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10777	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10778	}
10779	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10780	}
10781
10782	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10783	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
10784	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10785
10786	#if HC_ARCH_BITS == 64
10787	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10788	#endif
10789	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
10790	/* The 64-on-32 switcher maintains VMCS-launch state on its own
10791	and we need to leave it alone here. */
10792	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
10793	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10794	#else
10795	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10796	#endif
10797	#ifdef VBOX_STRICT
10798	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10799	#endif
10800	Assert(!ASMIntAreEnabled());
10801	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10802	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10803
10804	/*
10805	* Save the basic VM-exit reason and check if the VM-entry failed.
10806	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10807	*/
10808	uint32_t uExitReason;
10809	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
10810	AssertRC(rc);
10811	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
10812	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
10813
10814	/*
10815	* Check if VMLAUNCH/VMRESUME succeeded.
10816	* If this failed, we cause a guru meditation and cease further execution.
10817	*
10818	* However, if we are executing a nested-guest we might fail if we use the
10819	* fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
10820	*/
10821	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
10822	{
10823	/*
10824	* Update the VM-exit history array here even if the VM-entry failed due to:
10825	* - Invalid guest state.
10826	* - MSR loading.
10827	* - Machine-check event.
10828	*
10829	* In any of the above cases we will still have a "valid" VM-exit reason
10830	* despite @a fVMEntryFailed being false.
10831	*
10832	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
10833	*
10834	* Note! We don't have CS or RIP at this point. Will probably address that later
10835	* by amending the history entry added here.
10836	*/
10837	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
10838	UINT64_MAX, uHostTsc);
10839
10840	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
10841	{
10842	VMMRZCallRing3Enable(pVCpu);
10843
10844	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10845	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10846
10847	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
10848	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10849	AssertRC(rc);
10850	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
10851	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
10852	AssertRC(rc);
10853	#else
10854	/*
10855	* Import the guest-interruptibility state always as we need it while evaluating
10856	* injecting events on re-entry.
10857	*
10858	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
10859	* checking for real-mode while exporting the state because all bits that cause
10860	* mode changes wrt CR0 are intercepted.
10861	*/
10862	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
10863	AssertRC(rc);
10864	#endif
10865
10866	/*
10867	* Sync the TPR shadow with our APIC state.
10868	*/
10869	if ( !pVmxTransient->fIsNestedGuest
10870	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
10871	{
10872	Assert(pVmcsInfo->pbVirtApic);
10873	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
10874	{
10875	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
10876	AssertRC(rc);
10877	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
10878	}
10879	}
10880
10881	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10882	return;
10883	}
10884	}
10885	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10886	else if (pVmxTransient->fIsNestedGuest)
10887	{
10888	# if 0
10889	/*
10890	* Copy the VM-instruction error field to the guest VMCS.
10891	*/
10892	/** @todo NSTVMX: Verify we're using the fast path. */
10893	uint32_t u32RoVmInstrError;
10894	rc = VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &u32RoVmInstrError);
10895	AssertRCReturn(rc, rc);
10896	PVMXVVMCS pGstVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10897	pGstVmcs->u32RoVmInstrError = u32RoVmInstrError;
10898	/** @todo NSTVMX: Advance guest RIP and other fast path related restoration. */
10899	# else
10900	AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
10901	# endif
10902	}
10903	#endif
10904	else
10905	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
10906
10907	VMMRZCallRing3Enable(pVCpu);
10908	}
10909
10910
10911	/**
10912	* Runs the guest code using hardware-assisted VMX the normal way.
10913	*
10914	* @returns VBox status code.
10915	* @param pVCpu The cross context virtual CPU structure.
10916	* @param pcLoops Pointer to the number of executed loops.
10917	*/
10918	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
10919	{
10920	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10921	Assert(pcLoops);
10922	Assert(*pcLoops <= cMaxResumeLoops);
10923
10924	VMXTRANSIENT VmxTransient;
10925	RT_ZERO(VmxTransient);
10926	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10927
10928	/* Paranoia. */
10929	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
10930	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10931
10932	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10933	for (;;)
10934	{
10935	Assert(!HMR0SuspendPending());
10936	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10937	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10938
10939	/*
10940	* Preparatory work for running nested-guest code, this may force us to
10941	* return to ring-3.
10942	*
10943	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10944	*/
10945	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10946	if (rcStrict != VINF_SUCCESS)
10947	break;
10948
10949	/* Interrupts are disabled at this point! */
10950	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10951	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10952	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10953	/* Interrupts are re-enabled at this point! */
10954
10955	/*
10956	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10957	*/
10958	if (RT_SUCCESS(rcRun))
10959	{ /* very likely */ }
10960	else
10961	{
10962	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10963	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10964	return rcRun;
10965	}
10966
10967	/*
10968	* Profile the VM-exit.
10969	*/
10970	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10971	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10972	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10973	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10974	HMVMX_START_EXIT_DISPATCH_PROF();
10975
10976	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10977
10978	/*
10979	* Handle the VM-exit.
10980	*/
10981	#ifdef HMVMX_USE_FUNCTION_TABLE
10982	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
10983	#else
10984	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
10985	#endif
10986	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10987	if (rcStrict == VINF_SUCCESS)
10988	{
10989	if (++(*pcLoops) <= cMaxResumeLoops)
10990	continue;
10991	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10992	rcStrict = VINF_EM_RAW_INTERRUPT;
10993	}
10994	break;
10995	}
10996
10997	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10998	return rcStrict;
10999	}
11000
11001	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11002	/**
11003	* Runs the nested-guest code using hardware-assisted VMX.
11004	*
11005	* @returns VBox status code.
11006	* @param pVCpu The cross context virtual CPU structure.
11007	* @param pcLoops Pointer to the number of executed loops.
11008	*
11009	* @sa hmR0VmxRunGuestCodeNormal.
11010	*/
11011	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
11012	{
11013	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11014	Assert(pcLoops);
11015	Assert(*pcLoops <= cMaxResumeLoops);
11016
11017	VMXTRANSIENT VmxTransient;
11018	RT_ZERO(VmxTransient);
11019	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11020	VmxTransient.fIsNestedGuest = true;
11021
11022	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11023	for (;;)
11024	{
11025	Assert(!HMR0SuspendPending());
11026	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11027	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11028
11029	/*
11030	* Preparatory work for running guest code, this may force us to
11031	* return to ring-3.
11032	*
11033	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11034	*/
11035	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11036	if (rcStrict != VINF_SUCCESS)
11037	break;
11038
11039	/* Interrupts are disabled at this point! */
11040	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11041	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11042	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11043	/* Interrupts are re-enabled at this point! */
11044
11045	/*
11046	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11047	*/
11048	if (RT_SUCCESS(rcRun))
11049	{ /* very likely */ }
11050	else
11051	{
11052	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11053	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11054	return rcRun;
11055	}
11056
11057	/*
11058	* Profile the VM-exit.
11059	*/
11060	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11061	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11062	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11063	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11064	HMVMX_START_EXIT_DISPATCH_PROF();
11065
11066	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11067
11068	/*
11069	* Handle the VM-exit.
11070	*/
11071	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
11072	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11073	if ( rcStrict == VINF_SUCCESS
11074	&& CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11075	{
11076	if (++(*pcLoops) <= cMaxResumeLoops)
11077	continue;
11078	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11079	rcStrict = VINF_EM_RAW_INTERRUPT;
11080	}
11081	break;
11082	}
11083
11084	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11085	return rcStrict;
11086	}
11087	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11088
11089
11090	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
11091	* probes.
11092	*
11093	* The following few functions and associated structure contains the bloat
11094	* necessary for providing detailed debug events and dtrace probes as well as
11095	* reliable host side single stepping. This works on the principle of
11096	* "subclassing" the normal execution loop and workers. We replace the loop
11097	* method completely and override selected helpers to add necessary adjustments
11098	* to their core operation.
11099	*
11100	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11101	* any performance for debug and analysis features.
11102	*
11103	* @{
11104	*/
11105
11106	/**
11107	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11108	* the debug run loop.
11109	*/
11110	typedef struct VMXRUNDBGSTATE
11111	{
11112	/** The RIP we started executing at. This is for detecting that we stepped. */
11113	uint64_t uRipStart;
11114	/** The CS we started executing with. */
11115	uint16_t uCsStart;
11116
11117	/** Whether we've actually modified the 1st execution control field. */
11118	bool fModifiedProcCtls : 1;
11119	/** Whether we've actually modified the 2nd execution control field. */
11120	bool fModifiedProcCtls2 : 1;
11121	/** Whether we've actually modified the exception bitmap. */
11122	bool fModifiedXcptBitmap : 1;
11123
11124	/** We desire the modified the CR0 mask to be cleared. */
11125	bool fClearCr0Mask : 1;
11126	/** We desire the modified the CR4 mask to be cleared. */
11127	bool fClearCr4Mask : 1;
11128	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11129	uint32_t fCpe1Extra;
11130	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11131	uint32_t fCpe1Unwanted;
11132	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11133	uint32_t fCpe2Extra;
11134	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11135	uint32_t bmXcptExtra;
11136	/** The sequence number of the Dtrace provider settings the state was
11137	* configured against. */
11138	uint32_t uDtraceSettingsSeqNo;
11139	/** VM-exits to check (one bit per VM-exit). */
11140	uint32_t bmExitsToCheck[3];
11141
11142	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11143	uint32_t fProcCtlsInitial;
11144	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11145	uint32_t fProcCtls2Initial;
11146	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11147	uint32_t bmXcptInitial;
11148	} VMXRUNDBGSTATE;
11149	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11150	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11151
11152
11153	/**
11154	* Initializes the VMXRUNDBGSTATE structure.
11155	*
11156	* @param pVCpu The cross context virtual CPU structure of the
11157	* calling EMT.
11158	* @param pVmxTransient The VMX-transient structure.
11159	* @param pDbgState The debug state to initialize.
11160	*/
11161	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11162	{
11163	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11164	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11165
11166	pDbgState->fModifiedProcCtls = false;
11167	pDbgState->fModifiedProcCtls2 = false;
11168	pDbgState->fModifiedXcptBitmap = false;
11169	pDbgState->fClearCr0Mask = false;
11170	pDbgState->fClearCr4Mask = false;
11171	pDbgState->fCpe1Extra = 0;
11172	pDbgState->fCpe1Unwanted = 0;
11173	pDbgState->fCpe2Extra = 0;
11174	pDbgState->bmXcptExtra = 0;
11175	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11176	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11177	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11178	}
11179
11180
11181	/**
11182	* Updates the VMSC fields with changes requested by @a pDbgState.
11183	*
11184	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11185	* immediately before executing guest code, i.e. when interrupts are disabled.
11186	* We don't check status codes here as we cannot easily assert or return in the
11187	* latter case.
11188	*
11189	* @param pVCpu The cross context virtual CPU structure.
11190	* @param pVmxTransient The VMX-transient structure.
11191	* @param pDbgState The debug state.
11192	*/
11193	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11194	{
11195	/*
11196	* Ensure desired flags in VMCS control fields are set.
11197	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11198	*
11199	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11200	* there should be no stale data in pCtx at this point.
11201	*/
11202	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11203	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11204	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11205	{
11206	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11207	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11208	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11209	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11210	pDbgState->fModifiedProcCtls = true;
11211	}
11212
11213	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11214	{
11215	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11216	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11217	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11218	pDbgState->fModifiedProcCtls2 = true;
11219	}
11220
11221	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11222	{
11223	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11224	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11225	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11226	pDbgState->fModifiedXcptBitmap = true;
11227	}
11228
11229	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11230	{
11231	pVmcsInfo->u64Cr0Mask = 0;
11232	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
11233	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11234	}
11235
11236	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11237	{
11238	pVmcsInfo->u64Cr4Mask = 0;
11239	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
11240	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11241	}
11242
11243	NOREF(pVCpu);
11244	}
11245
11246
11247	/**
11248	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11249	* re-entry next time around.
11250	*
11251	* @returns Strict VBox status code (i.e. informational status codes too).
11252	* @param pVCpu The cross context virtual CPU structure.
11253	* @param pVmxTransient The VMX-transient structure.
11254	* @param pDbgState The debug state.
11255	* @param rcStrict The return code from executing the guest using single
11256	* stepping.
11257	*/
11258	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11259	VBOXSTRICTRC rcStrict)
11260	{
11261	/*
11262	* Restore VM-exit control settings as we may not reenter this function the
11263	* next time around.
11264	*/
11265	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11266
11267	/* We reload the initial value, trigger what we can of recalculations the
11268	next time around. From the looks of things, that's all that's required atm. */
11269	if (pDbgState->fModifiedProcCtls)
11270	{
11271	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11272	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11273	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11274	AssertRCReturn(rc2, rc2);
11275	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11276	}
11277
11278	/* We're currently the only ones messing with this one, so just restore the
11279	cached value and reload the field. */
11280	if ( pDbgState->fModifiedProcCtls2
11281	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11282	{
11283	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11284	AssertRCReturn(rc2, rc2);
11285	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11286	}
11287
11288	/* If we've modified the exception bitmap, we restore it and trigger
11289	reloading and partial recalculation the next time around. */
11290	if (pDbgState->fModifiedXcptBitmap)
11291	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11292
11293	return rcStrict;
11294	}
11295
11296
11297	/**
11298	* Configures VM-exit controls for current DBGF and DTrace settings.
11299	*
11300	* This updates @a pDbgState and the VMCS execution control fields to reflect
11301	* the necessary VM-exits demanded by DBGF and DTrace.
11302	*
11303	* @param pVCpu The cross context virtual CPU structure.
11304	* @param pVmxTransient The VMX-transient structure. May update
11305	* fUpdatedTscOffsettingAndPreemptTimer.
11306	* @param pDbgState The debug state.
11307	*/
11308	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11309	{
11310	/*
11311	* Take down the dtrace serial number so we can spot changes.
11312	*/
11313	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11314	ASMCompilerBarrier();
11315
11316	/*
11317	* We'll rebuild most of the middle block of data members (holding the
11318	* current settings) as we go along here, so start by clearing it all.
11319	*/
11320	pDbgState->bmXcptExtra = 0;
11321	pDbgState->fCpe1Extra = 0;
11322	pDbgState->fCpe1Unwanted = 0;
11323	pDbgState->fCpe2Extra = 0;
11324	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11325	pDbgState->bmExitsToCheck[i] = 0;
11326
11327	/*
11328	* Software interrupts (INT XXh) - no idea how to trigger these...
11329	*/
11330	PVM pVM = pVCpu->CTX_SUFF(pVM);
11331	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11332	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11333	{
11334	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11335	}
11336
11337	/*
11338	* INT3 breakpoints - triggered by #BP exceptions.
11339	*/
11340	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11341	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11342
11343	/*
11344	* Exception bitmap and XCPT events+probes.
11345	*/
11346	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11347	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11348	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11349
11350	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11351	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11352	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11353	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11354	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11355	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11356	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11357	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11358	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11359	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11360	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11361	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11362	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11363	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11364	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11365	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11366	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11367	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11368
11369	if (pDbgState->bmXcptExtra)
11370	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11371
11372	/*
11373	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11374	*
11375	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11376	* So, when adding/changing/removing please don't forget to update it.
11377	*
11378	* Some of the macros are picking up local variables to save horizontal space,
11379	* (being able to see it in a table is the lesser evil here).
11380	*/
11381	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11382	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11383	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11384	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11385	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11386	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11387	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11388	} else do { } while (0)
11389	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11390	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11391	{ \
11392	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11393	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11394	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11395	} else do { } while (0)
11396	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11397	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11398	{ \
11399	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11400	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11401	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11402	} else do { } while (0)
11403	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11404	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11405	{ \
11406	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11407	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11408	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11409	} else do { } while (0)
11410
11411	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11412	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11413	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11414	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11415	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11416
11417	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11418	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11419	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11420	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11421	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11422	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11423	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11424	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11425	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11426	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11427	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11428	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11429	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11430	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11431	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11432	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11433	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11434	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11435	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11436	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11437	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11438	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11439	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11440	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11441	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11442	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11443	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11444	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11445	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11446	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11447	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11448	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11449	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11450	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11451	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11452	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11453
11454	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11455	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11456	{
11457	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11458	\| CPUMCTX_EXTRN_APIC_TPR);
11459	AssertRC(rc);
11460
11461	#if 0 /** @todo fix me */
11462	pDbgState->fClearCr0Mask = true;
11463	pDbgState->fClearCr4Mask = true;
11464	#endif
11465	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11466	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11467	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11468	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11469	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11470	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11471	require clearing here and in the loop if we start using it. */
11472	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11473	}
11474	else
11475	{
11476	if (pDbgState->fClearCr0Mask)
11477	{
11478	pDbgState->fClearCr0Mask = false;
11479	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11480	}
11481	if (pDbgState->fClearCr4Mask)
11482	{
11483	pDbgState->fClearCr4Mask = false;
11484	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11485	}
11486	}
11487	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11488	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11489
11490	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11491	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11492	{
11493	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11494	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11495	}
11496	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11497	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11498
11499	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11500	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11501	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11502	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11503	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11504	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11505	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11506	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11507	#if 0 /** @todo too slow, fix handler. */
11508	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11509	#endif
11510	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11511
11512	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11513	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11514	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11515	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11516	{
11517	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11518	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11519	}
11520	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11521	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11522	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11523	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11524
11525	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11526	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11527	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11528	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11529	{
11530	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11531	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11532	}
11533	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11534	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11535	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11536	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11537
11538	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11539	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11540	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11541	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11542	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11543	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11544	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11545	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11546	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11547	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11548	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11549	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11550	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11551	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11552	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11553	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11554	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11555	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11556	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11557	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11558	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11559	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11560
11561	#undef IS_EITHER_ENABLED
11562	#undef SET_ONLY_XBM_IF_EITHER_EN
11563	#undef SET_CPE1_XBM_IF_EITHER_EN
11564	#undef SET_CPEU_XBM_IF_EITHER_EN
11565	#undef SET_CPE2_XBM_IF_EITHER_EN
11566
11567	/*
11568	* Sanitize the control stuff.
11569	*/
11570	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11571	if (pDbgState->fCpe2Extra)
11572	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11573	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11574	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11575	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11576	{
11577	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11578	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11579	}
11580
11581	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11582	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11583	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11584	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11585	}
11586
11587
11588	/**
11589	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11590	* appropriate.
11591	*
11592	* The caller has checked the VM-exit against the
11593	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11594	* already, so we don't have to do that either.
11595	*
11596	* @returns Strict VBox status code (i.e. informational status codes too).
11597	* @param pVCpu The cross context virtual CPU structure.
11598	* @param pVmxTransient The VMX-transient structure.
11599	* @param uExitReason The VM-exit reason.
11600	*
11601	* @remarks The name of this function is displayed by dtrace, so keep it short
11602	* and to the point. No longer than 33 chars long, please.
11603	*/
11604	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11605	{
11606	/*
11607	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11608	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11609	*
11610	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11611	* does. Must add/change/remove both places. Same ordering, please.
11612	*
11613	* Added/removed events must also be reflected in the next section
11614	* where we dispatch dtrace events.
11615	*/
11616	bool fDtrace1 = false;
11617	bool fDtrace2 = false;
11618	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11619	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11620	uint32_t uEventArg = 0;
11621	#define SET_EXIT(a_EventSubName) \
11622	do { \
11623	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11624	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11625	} while (0)
11626	#define SET_BOTH(a_EventSubName) \
11627	do { \
11628	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11629	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11630	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11631	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11632	} while (0)
11633	switch (uExitReason)
11634	{
11635	case VMX_EXIT_MTF:
11636	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11637
11638	case VMX_EXIT_XCPT_OR_NMI:
11639	{
11640	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11641	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11642	{
11643	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11644	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11645	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11646	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11647	{
11648	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11649	{
11650	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11651	uEventArg = pVmxTransient->uExitIntErrorCode;
11652	}
11653	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11654	switch (enmEvent1)
11655	{
11656	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11657	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11658	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11659	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11660	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11661	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11662	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11663	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11664	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11665	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11666	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11667	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11668	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11669	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11670	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11671	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11672	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11673	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11674	default: break;
11675	}
11676	}
11677	else
11678	AssertFailed();
11679	break;
11680
11681	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11682	uEventArg = idxVector;
11683	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11684	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11685	break;
11686	}
11687	break;
11688	}
11689
11690	case VMX_EXIT_TRIPLE_FAULT:
11691	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11692	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11693	break;
11694	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11695	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11696	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11697	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11698	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11699
11700	/* Instruction specific VM-exits: */
11701	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11702	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11703	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11704	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11705	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11706	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11707	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11708	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11709	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11710	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11711	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11712	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11713	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11714	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11715	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11716	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11717	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11718	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11719	case VMX_EXIT_MOV_CRX:
11720	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11721	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11722	SET_BOTH(CRX_READ);
11723	else
11724	SET_BOTH(CRX_WRITE);
11725	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11726	break;
11727	case VMX_EXIT_MOV_DRX:
11728	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11729	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11730	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11731	SET_BOTH(DRX_READ);
11732	else
11733	SET_BOTH(DRX_WRITE);
11734	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11735	break;
11736	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11737	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11738	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11739	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11740	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11741	case VMX_EXIT_GDTR_IDTR_ACCESS:
11742	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11743	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11744	{
11745	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11746	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11747	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11748	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11749	}
11750	break;
11751
11752	case VMX_EXIT_LDTR_TR_ACCESS:
11753	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11754	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11755	{
11756	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11757	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11758	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11759	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11760	}
11761	break;
11762
11763	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11764	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11765	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11766	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11767	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11768	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11769	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11770	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11771	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11772	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11773	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11774
11775	/* Events that aren't relevant at this point. */
11776	case VMX_EXIT_EXT_INT:
11777	case VMX_EXIT_INT_WINDOW:
11778	case VMX_EXIT_NMI_WINDOW:
11779	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11780	case VMX_EXIT_PREEMPT_TIMER:
11781	case VMX_EXIT_IO_INSTR:
11782	break;
11783
11784	/* Errors and unexpected events. */
11785	case VMX_EXIT_INIT_SIGNAL:
11786	case VMX_EXIT_SIPI:
11787	case VMX_EXIT_IO_SMI:
11788	case VMX_EXIT_SMI:
11789	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11790	case VMX_EXIT_ERR_MSR_LOAD:
11791	case VMX_EXIT_ERR_MACHINE_CHECK:
11792	case VMX_EXIT_PML_FULL:
11793	case VMX_EXIT_VIRTUALIZED_EOI:
11794	break;
11795
11796	default:
11797	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11798	break;
11799	}
11800	#undef SET_BOTH
11801	#undef SET_EXIT
11802
11803	/*
11804	* Dtrace tracepoints go first. We do them here at once so we don't
11805	* have to copy the guest state saving and stuff a few dozen times.
11806	* Down side is that we've got to repeat the switch, though this time
11807	* we use enmEvent since the probes are a subset of what DBGF does.
11808	*/
11809	if (fDtrace1 \|\| fDtrace2)
11810	{
11811	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11812	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11813	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11814	switch (enmEvent1)
11815	{
11816	/** @todo consider which extra parameters would be helpful for each probe. */
11817	case DBGFEVENT_END: break;
11818	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11819	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11820	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11821	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11822	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11823	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11824	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11825	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11826	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
11827	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
11828	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
11829	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
11830	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
11831	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
11832	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
11833	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
11834	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
11835	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
11836	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11837	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11838	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
11839	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
11840	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
11841	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
11842	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
11843	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
11844	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
11845	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11846	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11847	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11848	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11849	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11850	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
11851	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11852	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
11853	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
11854	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
11855	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
11856	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
11857	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
11858	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
11859	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
11860	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
11861	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
11862	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
11863	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
11864	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
11865	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
11866	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
11867	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
11868	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
11869	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
11870	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
11871	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
11872	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
11873	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
11874	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
11875	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
11876	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
11877	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
11878	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
11879	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
11880	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
11881	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
11882	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
11883	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
11884	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
11885	}
11886	switch (enmEvent2)
11887	{
11888	/** @todo consider which extra parameters would be helpful for each probe. */
11889	case DBGFEVENT_END: break;
11890	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
11891	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11892	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
11893	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
11894	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
11895	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
11896	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
11897	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
11898	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
11899	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11900	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11901	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11902	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11903	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11904	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
11905	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11906	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
11907	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
11908	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
11909	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
11910	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
11911	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
11912	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
11913	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
11914	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
11915	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
11916	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
11917	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
11918	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
11919	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
11920	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
11921	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
11922	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
11923	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
11924	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
11925	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
11926	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
11927	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
11928	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
11929	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
11930	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
11931	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
11932	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
11933	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
11934	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
11935	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
11936	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
11937	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
11938	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
11939	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
11940	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
11941	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
11942	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
11943	}
11944	}
11945
11946	/*
11947	* Fire of the DBGF event, if enabled (our check here is just a quick one,
11948	* the DBGF call will do a full check).
11949	*
11950	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
11951	* Note! If we have to events, we prioritize the first, i.e. the instruction
11952	* one, in order to avoid event nesting.
11953	*/
11954	PVM pVM = pVCpu->CTX_SUFF(pVM);
11955	if ( enmEvent1 != DBGFEVENT_END
11956	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
11957	{
11958	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11959	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
11960	if (rcStrict != VINF_SUCCESS)
11961	return rcStrict;
11962	}
11963	else if ( enmEvent2 != DBGFEVENT_END
11964	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
11965	{
11966	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11967	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
11968	if (rcStrict != VINF_SUCCESS)
11969	return rcStrict;
11970	}
11971
11972	return VINF_SUCCESS;
11973	}
11974
11975
11976	/**
11977	* Single-stepping VM-exit filtering.
11978	*
11979	* This is preprocessing the VM-exits and deciding whether we've gotten far
11980	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
11981	* handling is performed.
11982	*
11983	* @returns Strict VBox status code (i.e. informational status codes too).
11984	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
11985	* @param pVmxTransient The VMX-transient structure.
11986	* @param pDbgState The debug state.
11987	*/
11988	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11989	{
11990	/*
11991	* Expensive (saves context) generic dtrace VM-exit probe.
11992	*/
11993	uint32_t const uExitReason = pVmxTransient->uExitReason;
11994	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
11995	{ /* more likely */ }
11996	else
11997	{
11998	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11999	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12000	AssertRC(rc);
12001	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
12002	}
12003
12004	/*
12005	* Check for host NMI, just to get that out of the way.
12006	*/
12007	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
12008	{ /* normally likely */ }
12009	else
12010	{
12011	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12012	AssertRCReturn(rc2, rc2);
12013	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12014	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12015	return hmR0VmxExitHostNmi(pVCpu);
12016	}
12017
12018	/*
12019	* Check for single stepping event if we're stepping.
12020	*/
12021	if (pVCpu->hm.s.fSingleInstruction)
12022	{
12023	switch (uExitReason)
12024	{
12025	case VMX_EXIT_MTF:
12026	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12027
12028	/* Various events: */
12029	case VMX_EXIT_XCPT_OR_NMI:
12030	case VMX_EXIT_EXT_INT:
12031	case VMX_EXIT_TRIPLE_FAULT:
12032	case VMX_EXIT_INT_WINDOW:
12033	case VMX_EXIT_NMI_WINDOW:
12034	case VMX_EXIT_TASK_SWITCH:
12035	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12036	case VMX_EXIT_APIC_ACCESS:
12037	case VMX_EXIT_EPT_VIOLATION:
12038	case VMX_EXIT_EPT_MISCONFIG:
12039	case VMX_EXIT_PREEMPT_TIMER:
12040
12041	/* Instruction specific VM-exits: */
12042	case VMX_EXIT_CPUID:
12043	case VMX_EXIT_GETSEC:
12044	case VMX_EXIT_HLT:
12045	case VMX_EXIT_INVD:
12046	case VMX_EXIT_INVLPG:
12047	case VMX_EXIT_RDPMC:
12048	case VMX_EXIT_RDTSC:
12049	case VMX_EXIT_RSM:
12050	case VMX_EXIT_VMCALL:
12051	case VMX_EXIT_VMCLEAR:
12052	case VMX_EXIT_VMLAUNCH:
12053	case VMX_EXIT_VMPTRLD:
12054	case VMX_EXIT_VMPTRST:
12055	case VMX_EXIT_VMREAD:
12056	case VMX_EXIT_VMRESUME:
12057	case VMX_EXIT_VMWRITE:
12058	case VMX_EXIT_VMXOFF:
12059	case VMX_EXIT_VMXON:
12060	case VMX_EXIT_MOV_CRX:
12061	case VMX_EXIT_MOV_DRX:
12062	case VMX_EXIT_IO_INSTR:
12063	case VMX_EXIT_RDMSR:
12064	case VMX_EXIT_WRMSR:
12065	case VMX_EXIT_MWAIT:
12066	case VMX_EXIT_MONITOR:
12067	case VMX_EXIT_PAUSE:
12068	case VMX_EXIT_GDTR_IDTR_ACCESS:
12069	case VMX_EXIT_LDTR_TR_ACCESS:
12070	case VMX_EXIT_INVEPT:
12071	case VMX_EXIT_RDTSCP:
12072	case VMX_EXIT_INVVPID:
12073	case VMX_EXIT_WBINVD:
12074	case VMX_EXIT_XSETBV:
12075	case VMX_EXIT_RDRAND:
12076	case VMX_EXIT_INVPCID:
12077	case VMX_EXIT_VMFUNC:
12078	case VMX_EXIT_RDSEED:
12079	case VMX_EXIT_XSAVES:
12080	case VMX_EXIT_XRSTORS:
12081	{
12082	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12083	AssertRCReturn(rc, rc);
12084	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
12085	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
12086	return VINF_EM_DBG_STEPPED;
12087	break;
12088	}
12089
12090	/* Errors and unexpected events: */
12091	case VMX_EXIT_INIT_SIGNAL:
12092	case VMX_EXIT_SIPI:
12093	case VMX_EXIT_IO_SMI:
12094	case VMX_EXIT_SMI:
12095	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12096	case VMX_EXIT_ERR_MSR_LOAD:
12097	case VMX_EXIT_ERR_MACHINE_CHECK:
12098	case VMX_EXIT_PML_FULL:
12099	case VMX_EXIT_VIRTUALIZED_EOI:
12100	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12101	break;
12102
12103	default:
12104	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12105	break;
12106	}
12107	}
12108
12109	/*
12110	* Check for debugger event breakpoints and dtrace probes.
12111	*/
12112	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12113	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12114	{
12115	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12116	if (rcStrict != VINF_SUCCESS)
12117	return rcStrict;
12118	}
12119
12120	/*
12121	* Normal processing.
12122	*/
12123	#ifdef HMVMX_USE_FUNCTION_TABLE
12124	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
12125	#else
12126	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
12127	#endif
12128	}
12129
12130
12131	/**
12132	* Single steps guest code using hardware-assisted VMX.
12133	*
12134	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
12135	* but single-stepping through the hypervisor debugger.
12136	*
12137	* @returns Strict VBox status code (i.e. informational status codes too).
12138	* @param pVCpu The cross context virtual CPU structure.
12139	* @param pcLoops Pointer to the number of executed loops.
12140	*
12141	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
12142	*/
12143	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
12144	{
12145	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
12146	Assert(pcLoops);
12147	Assert(*pcLoops <= cMaxResumeLoops);
12148
12149	VMXTRANSIENT VmxTransient;
12150	RT_ZERO(VmxTransient);
12151	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
12152
12153	/* Set HMCPU indicators. */
12154	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
12155	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
12156	pVCpu->hm.s.fDebugWantRdTscExit = false;
12157	pVCpu->hm.s.fUsingDebugLoop = true;
12158
12159	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
12160	VMXRUNDBGSTATE DbgState;
12161	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
12162	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12163
12164	/*
12165	* The loop.
12166	*/
12167	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
12168	for (;;)
12169	{
12170	Assert(!HMR0SuspendPending());
12171	HMVMX_ASSERT_CPU_SAFE(pVCpu);
12172	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12173	bool fStepping = pVCpu->hm.s.fSingleInstruction;
12174
12175	/* Set up VM-execution controls the next two can respond to. */
12176	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12177
12178	/*
12179	* Preparatory work for running guest code, this may force us to
12180	* return to ring-3.
12181	*
12182	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12183	*/
12184	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12185	if (rcStrict != VINF_SUCCESS)
12186	break;
12187
12188	/* Interrupts are disabled at this point! */
12189	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12190
12191	/* Override any obnoxious code in the above two calls. */
12192	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12193
12194	/*
12195	* Finally execute the guest.
12196	*/
12197	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12198
12199	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12200	/* Interrupts are re-enabled at this point! */
12201
12202	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12203	if (RT_SUCCESS(rcRun))
12204	{ /* very likely */ }
12205	else
12206	{
12207	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12208	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12209	return rcRun;
12210	}
12211
12212	/* Profile the VM-exit. */
12213	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12214	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12215	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12216	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12217	HMVMX_START_EXIT_DISPATCH_PROF();
12218
12219	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12220
12221	/*
12222	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12223	*/
12224	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12225	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12226	if (rcStrict != VINF_SUCCESS)
12227	break;
12228	if (++(*pcLoops) > cMaxResumeLoops)
12229	{
12230	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12231	rcStrict = VINF_EM_RAW_INTERRUPT;
12232	break;
12233	}
12234
12235	/*
12236	* Stepping: Did the RIP change, if so, consider it a single step.
12237	* Otherwise, make sure one of the TFs gets set.
12238	*/
12239	if (fStepping)
12240	{
12241	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12242	AssertRC(rc);
12243	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12244	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12245	{
12246	rcStrict = VINF_EM_DBG_STEPPED;
12247	break;
12248	}
12249	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12250	}
12251
12252	/*
12253	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12254	*/
12255	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12256	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12257	}
12258
12259	/*
12260	* Clear the X86_EFL_TF if necessary.
12261	*/
12262	if (pVCpu->hm.s.fClearTrapFlag)
12263	{
12264	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12265	AssertRC(rc);
12266	pVCpu->hm.s.fClearTrapFlag = false;
12267	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12268	}
12269	/** @todo there seems to be issues with the resume flag when the monitor trap
12270	* flag is pending without being used. Seen early in bios init when
12271	* accessing APIC page in protected mode. */
12272
12273	/*
12274	* Restore VM-exit control settings as we may not re-enter this function the
12275	* next time around.
12276	*/
12277	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12278
12279	/* Restore HMCPU indicators. */
12280	pVCpu->hm.s.fUsingDebugLoop = false;
12281	pVCpu->hm.s.fDebugWantRdTscExit = false;
12282	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12283
12284	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12285	return rcStrict;
12286	}
12287
12288
12289	/** @} */
12290
12291
12292	/**
12293	* Checks if any expensive dtrace probes are enabled and we should go to the
12294	* debug loop.
12295	*
12296	* @returns true if we should use debug loop, false if not.
12297	*/
12298	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12299	{
12300	/* It's probably faster to OR the raw 32-bit counter variables together.
12301	Since the variables are in an array and the probes are next to one
12302	another (more or less), we have good locality. So, better read
12303	eight-nine cache lines ever time and only have one conditional, than
12304	128+ conditionals, right? */
12305	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12306	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12307	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12308	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12309	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12310	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12311	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12312	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12313	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12314	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12315	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12316	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12317	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12318	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12319	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12320	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12321	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12322	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12323	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12324	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12325	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12326	) != 0
12327	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12328	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12329	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12330	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12331	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12332	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12333	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12334	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12335	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12336	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12337	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12338	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12339	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12340	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12341	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12342	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12343	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12344	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12345	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12346	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12347	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12348	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12349	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12350	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12351	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12352	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12353	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12354	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12355	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12356	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12357	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12358	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12359	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12360	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12361	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12362	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12363	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12364	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12365	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12366	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12367	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12368	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12369	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12370	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12371	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12372	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12373	) != 0
12374	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12375	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12376	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12377	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12378	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12379	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12380	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12381	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12382	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12383	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12384	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12385	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12386	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12387	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12388	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12389	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12390	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12391	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12392	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12393	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12394	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12395	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12396	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12397	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12398	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12399	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12400	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12401	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12402	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12403	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12404	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12405	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12406	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12407	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12408	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12409	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12410	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12411	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12412	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12413	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12414	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12415	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12416	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12417	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12418	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12419	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12420	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12421	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12422	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12423	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12424	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12425	) != 0;
12426	}
12427
12428
12429	/**
12430	* Runs the guest using hardware-assisted VMX.
12431	*
12432	* @returns Strict VBox status code (i.e. informational status codes too).
12433	* @param pVCpu The cross context virtual CPU structure.
12434	*/
12435	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
12436	{
12437	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12438	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12439	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12440	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12441
12442	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
12443
12444	VBOXSTRICTRC rcStrict;
12445	uint32_t cLoops = 0;
12446	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12447	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12448	#else
12449	bool const fInNestedGuestMode = false;
12450	#endif
12451	if (!fInNestedGuestMode)
12452	{
12453	if ( !pVCpu->hm.s.fUseDebugLoop
12454	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12455	&& !DBGFIsStepping(pVCpu)
12456	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12457	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12458	else
12459	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12460	}
12461	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12462	else
12463	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
12464
12465	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12466	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12467	#endif
12468
12469	if (rcStrict == VERR_EM_INTERPRETER)
12470	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12471	else if (rcStrict == VINF_EM_RESET)
12472	rcStrict = VINF_EM_TRIPLE_FAULT;
12473
12474	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12475	if (RT_FAILURE(rc2))
12476	{
12477	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12478	rcStrict = rc2;
12479	}
12480	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12481	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12482	return rcStrict;
12483	}
12484
12485
12486	#ifndef HMVMX_USE_FUNCTION_TABLE
12487	/**
12488	* Handles a guest VM-exit from hardware-assisted VMX execution.
12489	*
12490	* @returns Strict VBox status code (i.e. informational status codes too).
12491	* @param pVCpu The cross context virtual CPU structure.
12492	* @param pVmxTransient The VMX-transient structure.
12493	*/
12494	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12495	{
12496	#ifdef DEBUG_ramshankar
12497	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12498	do { \
12499	if (a_fSave != 0) \
12500	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
12501	VBOXSTRICTRC rcStrict = a_CallExpr; \
12502	if (a_fSave != 0) \
12503	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12504	return rcStrict; \
12505	} while (0)
12506	#else
12507	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12508	#endif
12509	uint32_t const uExitReason = pVmxTransient->uExitReason;
12510	switch (uExitReason)
12511	{
12512	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12513	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12514	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12515	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12516	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12517	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12518	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12519	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12520	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12521	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12522	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12523	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12524	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12525	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12526	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12527	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12528	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12529	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12530	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12531	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12532	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12533	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12534	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12535	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12536	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12537	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12538	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12539	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12540	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12541	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12542	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12543	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12544	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12545	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12546	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12547	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12548	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12549	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12550	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12551	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12552	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
12553	case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient));
12554	#else
12555	case VMX_EXIT_VMCLEAR:
12556	case VMX_EXIT_VMLAUNCH:
12557	case VMX_EXIT_VMPTRLD:
12558	case VMX_EXIT_VMPTRST:
12559	case VMX_EXIT_VMREAD:
12560	case VMX_EXIT_VMRESUME:
12561	case VMX_EXIT_VMWRITE:
12562	case VMX_EXIT_VMXOFF:
12563	case VMX_EXIT_VMXON:
12564	case VMX_EXIT_INVVPID:
12565	case VMX_EXIT_INVEPT:
12566	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12567	#endif
12568
12569	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12570	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12571	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12572
12573	case VMX_EXIT_INIT_SIGNAL:
12574	case VMX_EXIT_SIPI:
12575	case VMX_EXIT_IO_SMI:
12576	case VMX_EXIT_SMI:
12577	case VMX_EXIT_ERR_MSR_LOAD:
12578	case VMX_EXIT_ERR_MACHINE_CHECK:
12579	case VMX_EXIT_PML_FULL:
12580	case VMX_EXIT_VIRTUALIZED_EOI:
12581	case VMX_EXIT_GDTR_IDTR_ACCESS:
12582	case VMX_EXIT_LDTR_TR_ACCESS:
12583	case VMX_EXIT_APIC_WRITE:
12584	case VMX_EXIT_RDRAND:
12585	case VMX_EXIT_RSM:
12586	case VMX_EXIT_VMFUNC:
12587	case VMX_EXIT_ENCLS:
12588	case VMX_EXIT_RDSEED:
12589	case VMX_EXIT_XSAVES:
12590	case VMX_EXIT_XRSTORS:
12591	case VMX_EXIT_UMWAIT:
12592	case VMX_EXIT_TPAUSE:
12593	default:
12594	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12595	}
12596	#undef VMEXIT_CALL_RET
12597	}
12598	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12599
12600
12601	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12602	/**
12603	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12604	*
12605	* @returns Strict VBox status code (i.e. informational status codes too).
12606	* @param pVCpu The cross context virtual CPU structure.
12607	* @param pVmxTransient The VMX-transient structure.
12608	*/
12609	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12610	{
12611	uint32_t const uExitReason = pVmxTransient->uExitReason;
12612	switch (uExitReason)
12613	{
12614	case VMX_EXIT_EPT_MISCONFIG: return hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
12615	case VMX_EXIT_EPT_VIOLATION: return hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
12616	case VMX_EXIT_IO_INSTR: return hmR0VmxExitIoInstrNested(pVCpu, pVmxTransient);
12617	case VMX_EXIT_HLT: return hmR0VmxExitHltNested(pVCpu, pVmxTransient);
12618	case VMX_EXIT_RDTSC: return hmR0VmxExitRdtscNested(pVCpu, pVmxTransient);
12619	case VMX_EXIT_RDTSCP: return hmR0VmxExitRdtscpNested(pVCpu, pVmxTransient);
12620
12621	/*
12622	* Instructions that cause VM-exits unconditionally.
12623	* - Provides VM-exit instruction length ONLY.
12624	*/
12625	case VMX_EXIT_CPUID:
12626	case VMX_EXIT_VMCALL:
12627	case VMX_EXIT_GETSEC:
12628	case VMX_EXIT_INVD:
12629	case VMX_EXIT_XSETBV:
12630	case VMX_EXIT_VMLAUNCH:
12631	case VMX_EXIT_VMRESUME:
12632	case VMX_EXIT_VMXOFF:
12633	return hmR0VmxExitInstrNested(pVCpu, pVmxTransient);
12634
12635	/*
12636	* Instructions that cause VM-exits unconditionally.
12637	* - Provides VM-exit instruction length.
12638	* - Provides VM-exit information.
12639	* - Optionally provides VM-exit qualification.
12640	*
12641	* Since VM-exit qualification is 0 for all VM-exits where it is not
12642	* applicable, reading and passing it to the guest should produce
12643	* defined behavior.
12644	*
12645	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
12646	*/
12647	case VMX_EXIT_INVEPT:
12648	case VMX_EXIT_INVVPID:
12649	case VMX_EXIT_VMCLEAR:
12650	case VMX_EXIT_VMPTRLD:
12651	case VMX_EXIT_VMPTRST:
12652	case VMX_EXIT_VMXON:
12653	return hmR0VmxExitInstrWithInfoNested(pVCpu, pVmxTransient);
12654
12655	/*
12656	* We shouldn't direct host physical interrupts to the nested-guest.
12657	*/
12658	case VMX_EXIT_EXT_INT:
12659	return hmR0VmxExitExtInt(pVCpu, pVmxTransient);
12660
12661	case VMX_EXIT_RDMSR: return hmR0VmxExitRdmsrNested(pVCpu, pVmxTransient);
12662	case VMX_EXIT_WRMSR: return hmR0VmxExitWrmsrNested(pVCpu, pVmxTransient);
12663	case VMX_EXIT_INVLPG: return hmR0VmxExitInvlpgNested(pVCpu, pVmxTransient);
12664	case VMX_EXIT_INVPCID: return hmR0VmxExitInvpcidNested(pVCpu, pVmxTransient);
12665	case VMX_EXIT_TASK_SWITCH: return hmR0VmxExitTaskSwitchNested(pVCpu, pVmxTransient);
12666	case VMX_EXIT_WBINVD: return hmR0VmxExitWbinvdNested(pVCpu, pVmxTransient);
12667	case VMX_EXIT_MTF: return hmR0VmxExitMtfNested(pVCpu, pVmxTransient);
12668	case VMX_EXIT_APIC_ACCESS: return hmR0VmxExitApicAccessNested(pVCpu, pVmxTransient);
12669
12670	/** @todo NSTVMX: APIC-access, Xcpt or NMI, Mov CRx. */
12671	case VMX_EXIT_XCPT_OR_NMI:
12672	{
12673	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12674	}
12675
12676	case VMX_EXIT_MOV_CRX: return hmR0VmxExitMovCRxNested(pVCpu, pVmxTransient);
12677	case VMX_EXIT_INT_WINDOW: return hmR0VmxExitIntWindowNested(pVCpu, pVmxTransient);
12678	case VMX_EXIT_TPR_BELOW_THRESHOLD: return hmR0VmxExitTprBelowThresholdNested(pVCpu, pVmxTransient);
12679	case VMX_EXIT_MWAIT: return hmR0VmxExitMwaitNested(pVCpu, pVmxTransient);
12680	case VMX_EXIT_MONITOR: return hmR0VmxExitMonitorNested(pVCpu, pVmxTransient);
12681	case VMX_EXIT_PAUSE: return hmR0VmxExitPauseNested(pVCpu, pVmxTransient);
12682
12683	case VMX_EXIT_PREEMPT_TIMER:
12684	{
12685	/** @todo NSTVMX: Preempt timer. */
12686	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12687	}
12688
12689	case VMX_EXIT_MOV_DRX: return hmR0VmxExitMovDRxNested(pVCpu, pVmxTransient);
12690	case VMX_EXIT_RDPMC: return hmR0VmxExitRdpmcNested(pVCpu, pVmxTransient);
12691
12692	case VMX_EXIT_GDTR_IDTR_ACCESS:
12693	case VMX_EXIT_LDTR_TR_ACCESS: return hmR0VmxExitXdtrAccessNested(pVCpu, pVmxTransient);
12694
12695	case VMX_EXIT_VMREAD:
12696	case VMX_EXIT_VMWRITE: return hmR0VmxExitVmreadVmwriteNested(pVCpu, pVmxTransient);
12697
12698	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFaultNested(pVCpu, pVmxTransient);
12699	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindowNested(pVCpu, pVmxTransient);
12700
12701	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12702	{
12703	/** @todo NSTVMX: Invalid guest state. */
12704	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12705	}
12706
12707	case VMX_EXIT_INIT_SIGNAL:
12708	case VMX_EXIT_SIPI:
12709	case VMX_EXIT_IO_SMI:
12710	case VMX_EXIT_SMI:
12711	case VMX_EXIT_ERR_MSR_LOAD:
12712	case VMX_EXIT_ERR_MACHINE_CHECK:
12713	case VMX_EXIT_PML_FULL:
12714	case VMX_EXIT_VIRTUALIZED_EOI:
12715	case VMX_EXIT_APIC_WRITE:
12716	case VMX_EXIT_RDRAND:
12717	case VMX_EXIT_RSM:
12718	case VMX_EXIT_VMFUNC:
12719	case VMX_EXIT_ENCLS:
12720	case VMX_EXIT_RDSEED:
12721	case VMX_EXIT_XSAVES:
12722	case VMX_EXIT_XRSTORS:
12723	case VMX_EXIT_UMWAIT:
12724	case VMX_EXIT_TPAUSE:
12725	default:
12726	{
12727	/** @todo NSTVMX: implement me! */
12728	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12729	}
12730	}
12731	}
12732	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12733
12734
12735	#ifdef VBOX_STRICT
12736	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
12737	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
12738	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
12739
12740	# define HMVMX_ASSERT_PREEMPT_CPUID() \
12741	do { \
12742	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
12743	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
12744	} while (0)
12745
12746	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12747	do { \
12748	AssertPtr((a_pVCpu)); \
12749	AssertPtr((a_pVmxTransient)); \
12750	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
12751	Assert((a_pVmxTransient)->pVmcsInfo); \
12752	Assert(ASMIntAreEnabled()); \
12753	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12754	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
12755	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
12756	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12757	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
12758	HMVMX_ASSERT_PREEMPT_CPUID(); \
12759	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12760	} while (0)
12761
12762	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12763	do { \
12764	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
12765	Assert((a_pVmxTransient)->fIsNestedGuest); \
12766	} while (0)
12767
12768	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12769	do { \
12770	Log4Func(("\n")); \
12771	} while (0)
12772	#else
12773	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12774	do { \
12775	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12776	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
12777	} while (0)
12778
12779	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12780	do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
12781
12782	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
12783	#endif
12784
12785
12786	/**
12787	* Advances the guest RIP by the specified number of bytes.
12788	*
12789	* @param pVCpu The cross context virtual CPU structure.
12790	* @param cbInstr Number of bytes to advance the RIP by.
12791	*
12792	* @remarks No-long-jump zone!!!
12793	*/
12794	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
12795	{
12796	/* Advance the RIP. */
12797	pVCpu->cpum.GstCtx.rip += cbInstr;
12798	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12799
12800	/* Update interrupt inhibition. */
12801	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
12802	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
12803	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12804	}
12805
12806
12807	/**
12808	* Advances the guest RIP after reading it from the VMCS.
12809	*
12810	* @returns VBox status code, no informational status codes.
12811	* @param pVCpu The cross context virtual CPU structure.
12812	* @param pVmxTransient The VMX-transient structure.
12813	*
12814	* @remarks No-long-jump zone!!!
12815	*/
12816	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12817	{
12818	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12819	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
12820	AssertRCReturn(rc, rc);
12821
12822	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
12823	return VINF_SUCCESS;
12824	}
12825
12826
12827	/**
12828	* Handle a condition that occurred while delivering an event through the guest
12829	* IDT.
12830	*
12831	* @returns Strict VBox status code (i.e. informational status codes too).
12832	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
12833	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
12834	* to continue execution of the guest which will delivery the \#DF.
12835	* @retval VINF_EM_RESET if we detected a triple-fault condition.
12836	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
12837	*
12838	* @param pVCpu The cross context virtual CPU structure.
12839	* @param pVmxTransient The VMX-transient structure.
12840	*
12841	* @remarks No-long-jump zone!!!
12842	*/
12843	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12844	{
12845	uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12846
12847	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12848	rc2 \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12849	AssertRCReturn(rc2, rc2);
12850
12851	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
12852	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12853	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12854	{
12855	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12856	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12857
12858	/*
12859	* If the event was a software interrupt (generated with INT n) or a software exception
12860	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
12861	* can handle the VM-exit and continue guest execution which will re-execute the
12862	* instruction rather than re-injecting the exception, as that can cause premature
12863	* trips to ring-3 before injection and involve TRPM which currently has no way of
12864	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
12865	* the problem).
12866	*/
12867	IEMXCPTRAISE enmRaise;
12868	IEMXCPTRAISEINFO fRaiseInfo;
12869	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
12870	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
12871	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
12872	{
12873	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
12874	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12875	}
12876	else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
12877	{
12878	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12879	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
12880	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
12881	/** @todo Make AssertMsgReturn as just AssertMsg later. */
12882	AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
12883	("Unexpected VM-exit interruption vector type %#x!\n", uExitVectorType), VERR_VMX_IPE_5);
12884
12885	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
12886
12887	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
12888	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
12889	{
12890	pVmxTransient->fVectoringPF = true;
12891	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12892	}
12893	}
12894	else
12895	{
12896	/*
12897	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
12898	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
12899	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
12900	*/
12901	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12902	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12903	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
12904	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12905	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12906	}
12907
12908	/*
12909	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
12910	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
12911	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
12912	* subsequent VM-entry would fail.
12913	*
12914	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
12915	*/
12916	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS)
12917	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12918	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
12919	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
12920	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
12921	{
12922	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
12923	}
12924
12925	switch (enmRaise)
12926	{
12927	case IEMXCPTRAISE_CURRENT_XCPT:
12928	{
12929	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
12930	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
12931	Assert(rcStrict == VINF_SUCCESS);
12932	break;
12933	}
12934
12935	case IEMXCPTRAISE_PREV_EVENT:
12936	{
12937	uint32_t u32ErrCode;
12938	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
12939	{
12940	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12941	AssertRCReturn(rc2, rc2);
12942	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12943	}
12944	else
12945	u32ErrCode = 0;
12946
12947	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
12948	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
12949	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12950	0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
12951
12952	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
12953	pVCpu->hm.s.Event.u32ErrCode));
12954	Assert(rcStrict == VINF_SUCCESS);
12955	break;
12956	}
12957
12958	case IEMXCPTRAISE_REEXEC_INSTR:
12959	Assert(rcStrict == VINF_SUCCESS);
12960	break;
12961
12962	case IEMXCPTRAISE_DOUBLE_FAULT:
12963	{
12964	/*
12965	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
12966	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
12967	*/
12968	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
12969	{
12970	pVmxTransient->fVectoringDoublePF = true;
12971	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
12972	pVCpu->cpum.GstCtx.cr2));
12973	rcStrict = VINF_SUCCESS;
12974	}
12975	else
12976	{
12977	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
12978	hmR0VmxSetPendingXcptDF(pVCpu);
12979	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
12980	uIdtVector, uExitVector));
12981	rcStrict = VINF_HM_DOUBLE_FAULT;
12982	}
12983	break;
12984	}
12985
12986	case IEMXCPTRAISE_TRIPLE_FAULT:
12987	{
12988	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
12989	rcStrict = VINF_EM_RESET;
12990	break;
12991	}
12992
12993	case IEMXCPTRAISE_CPU_HANG:
12994	{
12995	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
12996	rcStrict = VERR_EM_GUEST_CPU_HANG;
12997	break;
12998	}
12999
13000	default:
13001	{
13002	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
13003	rcStrict = VERR_VMX_IPE_2;
13004	break;
13005	}
13006	}
13007	}
13008	else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
13009	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
13010	&& uExitVector != X86_XCPT_DF
13011	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
13012	{
13013	/*
13014	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
13015	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
13016	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
13017	*/
13018	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
13019	{
13020	Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
13021	VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
13022	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
13023	}
13024	}
13025
13026	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
13027	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
13028	return rcStrict;
13029	}
13030
13031
13032	/** @name VM-exit handlers.
13033	* @{
13034	*/
13035	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13036	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
13037	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13038
13039	/**
13040	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
13041	*/
13042	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13043	{
13044	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13045	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
13046	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
13047	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
13048	return VINF_SUCCESS;
13049	return VINF_EM_RAW_INTERRUPT;
13050	}
13051
13052
13053	/**
13054	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
13055	*/
13056	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13057	{
13058	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13059	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
13060
13061	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13062	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13063	AssertRCReturn(rc, rc);
13064
13065	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
13066	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
13067	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
13068	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
13069
13070	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
13071	{
13072	/*
13073	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
13074	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
13075	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
13076	*
13077	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
13078	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
13079	*/
13080	return hmR0VmxExitHostNmi(pVCpu);
13081	}
13082
13083	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13084	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
13085	if (RT_UNLIKELY(rcStrict == VINF_SUCCESS))
13086	{ /* likely */ }
13087	else
13088	{
13089	if (rcStrict == VINF_HM_DOUBLE_FAULT)
13090	rcStrict = VINF_SUCCESS;
13091	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
13092	return rcStrict;
13093	}
13094
13095	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
13096	uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
13097	switch (uIntType)
13098	{
13099	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
13100	Assert(uVector == X86_XCPT_DB);
13101	RT_FALL_THRU();
13102	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
13103	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
13104	RT_FALL_THRU();
13105	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
13106	{
13107	/*
13108	* If there's any exception caused as a result of event injection, the resulting
13109	* secondary/final execption will be pending, we shall continue guest execution
13110	* after injecting the event. The page-fault case is complicated and we manually
13111	* handle any currently pending event in hmR0VmxExitXcptPF.
13112	*/
13113	if (!pVCpu->hm.s.Event.fPending)
13114	{ /* likely */ }
13115	else if (uVector != X86_XCPT_PF)
13116	{
13117	rcStrict = VINF_SUCCESS;
13118	break;
13119	}
13120
13121	switch (uVector)
13122	{
13123	case X86_XCPT_PF: rcStrict = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
13124	case X86_XCPT_GP: rcStrict = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
13125	case X86_XCPT_MF: rcStrict = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
13126	case X86_XCPT_DB: rcStrict = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
13127	case X86_XCPT_BP: rcStrict = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
13128	case X86_XCPT_AC: rcStrict = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
13129
13130	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13131	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13132	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
13133	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13134	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
13135	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13136	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
13137	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13138	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
13139	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13140	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
13141	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13142	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
13143	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13144	default:
13145	{
13146	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
13147	if (pVmcsInfo->RealMode.fRealOnV86Active)
13148	{
13149	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
13150	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
13151	Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
13152
13153	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
13154	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13155	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13156	AssertRCReturn(rc, rc);
13157	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
13158	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
13159	0 /* GCPtrFaultAddress */);
13160	rcStrict = VINF_SUCCESS;
13161	}
13162	else
13163	{
13164	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
13165	pVCpu->hm.s.u32HMError = uVector;
13166	rcStrict = VERR_VMX_UNEXPECTED_EXCEPTION;
13167	}
13168	break;
13169	}
13170	}
13171	break;
13172	}
13173
13174	default:
13175	{
13176	pVCpu->hm.s.u32HMError = uExitIntInfo;
13177	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
13178	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
13179	break;
13180	}
13181	}
13182	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
13183	return rcStrict;
13184	}
13185
13186
13187	/**
13188	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
13189	*/
13190	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13191	{
13192	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13193
13194	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
13195	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13196	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
13197	AssertRCReturn(rc, rc);
13198
13199	/* Evaluate and deliver pending events and resume guest execution. */
13200	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
13201	return VINF_SUCCESS;
13202	}
13203
13204
13205	/**
13206	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
13207	*/
13208	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13209	{
13210	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13211
13212	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13213	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
13214	{
13215	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
13216	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
13217	}
13218
13219	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS));
13220
13221	/*
13222	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
13223	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
13224	*/
13225	uint32_t fIntrState;
13226	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13227	AssertRCReturn(rc, rc);
13228	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
13229	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
13230	{
13231	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
13232	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
13233
13234	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
13235	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
13236	AssertRCReturn(rc, rc);
13237	}
13238
13239	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
13240	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
13241	AssertRCReturn(rc, rc);
13242
13243	/* Evaluate and deliver pending events and resume guest execution. */
13244	return VINF_SUCCESS;
13245	}
13246
13247
13248	/**
13249	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
13250	*/
13251	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13252	{
13253	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13254	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13255	}
13256
13257
13258	/**
13259	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
13260	*/
13261	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13262	{
13263	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13264	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13265	}
13266
13267
13268	/**
13269	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
13270	*/
13271	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13272	{
13273	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13274
13275	/*
13276	* Get the state we need and update the exit history entry.
13277	*/
13278	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13279	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13280	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13281	AssertRCReturn(rc, rc);
13282
13283	VBOXSTRICTRC rcStrict;
13284	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
13285	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
13286	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
13287	if (!pExitRec)
13288	{
13289	/*
13290	* Regular CPUID instruction execution.
13291	*/
13292	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
13293	if (rcStrict == VINF_SUCCESS)
13294	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13295	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13296	{
13297	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13298	rcStrict = VINF_SUCCESS;
13299	}
13300	}
13301	else
13302	{
13303	/*
13304	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
13305	*/
13306	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13307	AssertRCReturn(rc2, rc2);
13308
13309	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
13310	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
13311
13312	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
13313	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13314
13315	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
13316	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13317	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
13318	}
13319	return rcStrict;
13320	}
13321
13322
13323	/**
13324	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
13325	*/
13326	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13327	{
13328	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13329
13330	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13331	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
13332	AssertRCReturn(rc, rc);
13333
13334	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
13335	return VINF_EM_RAW_EMULATE_INSTR;
13336
13337	AssertMsgFailed(("hmR0VmxExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
13338	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
13339	}
13340
13341
13342	/**
13343	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
13344	*/
13345	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13346	{
13347	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13348
13349	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13350	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13351	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13352	AssertRCReturn(rc, rc);
13353
13354	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
13355	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13356	{
13357	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13358	we must reset offsetting on VM-entry. See @bugref{6634}. */
13359	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13360	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13361	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13362	}
13363	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13364	{
13365	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13366	rcStrict = VINF_SUCCESS;
13367	}
13368	return rcStrict;
13369	}
13370
13371
13372	/**
13373	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
13374	*/
13375	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13376	{
13377	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13378
13379	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13380	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
13381	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13382	AssertRCReturn(rc, rc);
13383
13384	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
13385	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13386	{
13387	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13388	we must reset offsetting on VM-reentry. See @bugref{6634}. */
13389	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13390	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13391	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13392	}
13393	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13394	{
13395	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13396	rcStrict = VINF_SUCCESS;
13397	}
13398	return rcStrict;
13399	}
13400
13401
13402	/**
13403	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
13404	*/
13405	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13406	{
13407	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13408
13409	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13410	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
13411	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13412	AssertRCReturn(rc, rc);
13413
13414	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13415	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13416	if (RT_LIKELY(rc == VINF_SUCCESS))
13417	{
13418	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13419	Assert(pVmxTransient->cbInstr == 2);
13420	}
13421	else
13422	{
13423	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
13424	rc = VERR_EM_INTERPRETER;
13425	}
13426	return rc;
13427	}
13428
13429
13430	/**
13431	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
13432	*/
13433	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13434	{
13435	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13436
13437	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
13438	if (EMAreHypercallInstructionsEnabled(pVCpu))
13439	{
13440	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13441	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
13442	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13443	AssertRCReturn(rc, rc);
13444
13445	/* Perform the hypercall. */
13446	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
13447	if (rcStrict == VINF_SUCCESS)
13448	{
13449	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13450	AssertRCReturn(rc, rc);
13451	}
13452	else
13453	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
13454	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
13455	\|\| RT_FAILURE(rcStrict));
13456
13457	/* If the hypercall changes anything other than guest's general-purpose registers,
13458	we would need to reload the guest changed bits here before VM-entry. */
13459	}
13460	else
13461	Log4Func(("Hypercalls not enabled\n"));
13462
13463	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
13464	if (RT_FAILURE(rcStrict))
13465	{
13466	hmR0VmxSetPendingXcptUD(pVCpu);
13467	rcStrict = VINF_SUCCESS;
13468	}
13469
13470	return rcStrict;
13471	}
13472
13473
13474	/**
13475	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
13476	*/
13477	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13478	{
13479	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13480	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
13481
13482	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13483	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13484	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13485	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13486	AssertRCReturn(rc, rc);
13487
13488	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
13489
13490	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
13491	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13492	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13493	{
13494	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13495	rcStrict = VINF_SUCCESS;
13496	}
13497	else
13498	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
13499	VBOXSTRICTRC_VAL(rcStrict)));
13500	return rcStrict;
13501	}
13502
13503
13504	/**
13505	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
13506	*/
13507	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13508	{
13509	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13510
13511	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13512	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
13513	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13514	AssertRCReturn(rc, rc);
13515
13516	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbInstr);
13517	if (rcStrict == VINF_SUCCESS)
13518	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13519	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13520	{
13521	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13522	rcStrict = VINF_SUCCESS;
13523	}
13524
13525	return rcStrict;
13526	}
13527
13528
13529	/**
13530	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
13531	*/
13532	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13533	{
13534	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13535
13536	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13537	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13538	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13539	AssertRCReturn(rc, rc);
13540
13541	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbInstr);
13542	if (RT_SUCCESS(rcStrict))
13543	{
13544	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13545	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
13546	rcStrict = VINF_SUCCESS;
13547	}
13548
13549	return rcStrict;
13550	}
13551
13552
13553	/**
13554	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
13555	* VM-exit.
13556	*/
13557	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13558	{
13559	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13560	return VINF_EM_RESET;
13561	}
13562
13563
13564	/**
13565	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
13566	*/
13567	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13568	{
13569	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13570
13571	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13572	AssertRCReturn(rc, rc);
13573
13574	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
13575	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
13576	rc = VINF_SUCCESS;
13577	else
13578	rc = VINF_EM_HALT;
13579
13580	if (rc != VINF_SUCCESS)
13581	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
13582	return rc;
13583	}
13584
13585
13586	/**
13587	* VM-exit handler for instructions that result in a \#UD exception delivered to
13588	* the guest.
13589	*/
13590	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13591	{
13592	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13593	hmR0VmxSetPendingXcptUD(pVCpu);
13594	return VINF_SUCCESS;
13595	}
13596
13597
13598	/**
13599	* VM-exit handler for expiry of the VMX-preemption timer.
13600	*/
13601	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13602	{
13603	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13604
13605	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
13606	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13607
13608	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
13609	PVM pVM = pVCpu->CTX_SUFF(pVM);
13610	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
13611	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
13612	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
13613	}
13614
13615
13616	/**
13617	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
13618	*/
13619	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13620	{
13621	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13622
13623	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13624	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13625	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
13626	AssertRCReturn(rc, rc);
13627
13628	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
13629	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13630	: HM_CHANGED_RAISED_XCPT_MASK);
13631
13632	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13633	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
13634
13635	return rcStrict;
13636	}
13637
13638
13639	/**
13640	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
13641	*/
13642	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13643	{
13644	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13645	/** @todo Use VM-exit instruction information. */
13646	return VERR_EM_INTERPRETER;
13647	}
13648
13649
13650	/**
13651	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
13652	* Error VM-exit.
13653	*/
13654	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13655	{
13656	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13657	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13658	AssertRCReturn(rc, rc);
13659
13660	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo);
13661	if (RT_FAILURE(rc))
13662	return rc;
13663
13664	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
13665	NOREF(uInvalidReason);
13666
13667	#ifdef VBOX_STRICT
13668	uint32_t fIntrState;
13669	RTHCUINTREG uHCReg;
13670	uint64_t u64Val;
13671	uint32_t u32Val;
13672	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
13673	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
13674	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
13675	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13676	AssertRCReturn(rc, rc);
13677
13678	Log4(("uInvalidReason %u\n", uInvalidReason));
13679	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
13680	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
13681	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
13682	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
13683
13684	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
13685	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
13686	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
13687	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
13688	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
13689	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13690	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
13691	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
13692	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
13693	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13694	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
13695	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
13696
13697	hmR0DumpRegs(pVCpu);
13698	#endif
13699
13700	return VERR_VMX_INVALID_GUEST_STATE;
13701	}
13702
13703	/**
13704	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
13705	*/
13706	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUnexpected(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13707	{
13708	/*
13709	* Cummulative notes of all recognized but unexpected VM-exits.
13710	*
13711	* 1. This does -not- cover scenarios like like a page-fault VM-exit occurring when
13712	* nested-paging is used.
13713	*
13714	* 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
13715	* emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
13716	* this function (and thereby stop VM execution) for handling such instructions.
13717	*
13718	*
13719	* VMX_EXIT_INIT_SIGNAL:
13720	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
13721	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
13722	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
13723	*
13724	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
13725	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
13726	* See Intel spec. "23.8 Restrictions on VMX operation".
13727	*
13728	* VMX_EXIT_SIPI:
13729	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
13730	* activity state is used. We don't make use of it as our guests don't have direct
13731	* access to the host local APIC.
13732	*
13733	* See Intel spec. 25.3 "Other Causes of VM-exits".
13734	*
13735	* VMX_EXIT_IO_SMI:
13736	* VMX_EXIT_SMI:
13737	* This can only happen if we support dual-monitor treatment of SMI, which can be
13738	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
13739	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
13740	* VMX root mode or receive an SMI. If we get here, something funny is going on.
13741	*
13742	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
13743	* See Intel spec. 25.3 "Other Causes of VM-Exits"
13744	*
13745	* VMX_EXIT_ERR_MSR_LOAD:
13746	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
13747	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
13748	* execution.
13749	*
13750	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
13751	*
13752	* VMX_EXIT_ERR_MACHINE_CHECK:
13753	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
13754	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
13755	* #MC exception abort class exception is raised. We thus cannot assume a
13756	* reasonable chance of continuing any sort of execution and we bail.
13757	*
13758	* See Intel spec. 15.1 "Machine-check Architecture".
13759	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
13760	*
13761	* VMX_EXIT_PML_FULL:
13762	* VMX_EXIT_VIRTUALIZED_EOI:
13763	* VMX_EXIT_APIC_WRITE:
13764	* We do not currently support any of these features and thus they are all unexpected
13765	* VM-exits.
13766	*
13767	* VMX_EXIT_GDTR_IDTR_ACCESS:
13768	* VMX_EXIT_LDTR_TR_ACCESS:
13769	* VMX_EXIT_RDRAND:
13770	* VMX_EXIT_RSM:
13771	* VMX_EXIT_VMFUNC:
13772	* VMX_EXIT_ENCLS:
13773	* VMX_EXIT_RDSEED:
13774	* VMX_EXIT_XSAVES:
13775	* VMX_EXIT_XRSTORS:
13776	* VMX_EXIT_UMWAIT:
13777	* VMX_EXIT_TPAUSE:
13778	* These VM-exits are -not- caused unconditionally by execution of the corresponding
13779	* instruction. Any VM-exit for these instructions indicate a hardware problem,
13780	* unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
13781	*
13782	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
13783	*/
13784	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13785	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
13786	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
13787	}
13788
13789
13790	/**
13791	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
13792	*/
13793	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13794	{
13795	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13796
13797	/** @todo Optimize this: We currently drag in in the whole MSR state
13798	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13799	* MSRs required. That would require changes to IEM and possibly CPUM too.
13800	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13801	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13802	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13803	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13804	switch (idMsr)
13805	{
13806	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13807	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13808	}
13809
13810	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13811	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13812	AssertRCReturn(rc, rc);
13813
13814	Log4Func(("ecx=%#RX32\n", idMsr));
13815
13816	#ifdef VBOX_STRICT
13817	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
13818	{
13819	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
13820	&& idMsr != MSR_K6_EFER)
13821	{
13822	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
13823	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
13824	}
13825	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13826	{
13827	Assert(pVmcsInfo->pvMsrBitmap);
13828	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13829	if (fMsrpm & VMXMSRPM_ALLOW_RD)
13830	{
13831	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
13832	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
13833	}
13834	}
13835	}
13836	#endif
13837
13838	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
13839	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
13840	if (rcStrict == VINF_SUCCESS)
13841	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13842	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
13843	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13844	{
13845	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13846	rcStrict = VINF_SUCCESS;
13847	}
13848	else
13849	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13850
13851	return rcStrict;
13852	}
13853
13854
13855	/**
13856	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
13857	*/
13858	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13859	{
13860	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13861
13862	/** @todo Optimize this: We currently drag in in the whole MSR state
13863	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13864	* MSRs required. That would require changes to IEM and possibly CPUM too.
13865	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13866	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13867	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13868
13869	/*
13870	* The FS and GS base MSRs are not part of the above all-MSRs mask.
13871	* Although we don't need to fetch the base as it will be overwritten shortly, while
13872	* loading guest-state we would also load the entire segment register including limit
13873	* and attributes and thus we need to load them here.
13874	*/
13875	switch (idMsr)
13876	{
13877	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13878	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13879	}
13880
13881	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13882	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13883	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13884	AssertRCReturn(rc, rc);
13885
13886	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
13887
13888	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
13889	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
13890
13891	if (rcStrict == VINF_SUCCESS)
13892	{
13893	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13894
13895	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
13896	if ( idMsr == MSR_IA32_APICBASE
13897	\|\| ( idMsr >= MSR_IA32_X2APIC_START
13898	&& idMsr <= MSR_IA32_X2APIC_END))
13899	{
13900	/*
13901	* We've already saved the APIC related guest-state (TPR) in post-run phase.
13902	* When full APIC register virtualization is implemented we'll have to make
13903	* sure APIC state is saved from the VMCS before IEM changes it.
13904	*/
13905	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
13906	}
13907	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
13908	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13909	else if (idMsr == MSR_K6_EFER)
13910	{
13911	/*
13912	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
13913	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
13914	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
13915	*/
13916	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
13917	}
13918
13919	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
13920	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
13921	{
13922	switch (idMsr)
13923	{
13924	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
13925	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
13926	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
13927	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
13928	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
13929	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
13930	default:
13931	{
13932	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
13933	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
13934	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13935	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
13936	break;
13937	}
13938	}
13939	}
13940	#ifdef VBOX_STRICT
13941	else
13942	{
13943	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
13944	switch (idMsr)
13945	{
13946	case MSR_IA32_SYSENTER_CS:
13947	case MSR_IA32_SYSENTER_EIP:
13948	case MSR_IA32_SYSENTER_ESP:
13949	case MSR_K8_FS_BASE:
13950	case MSR_K8_GS_BASE:
13951	{
13952	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
13953	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
13954	}
13955
13956	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
13957	default:
13958	{
13959	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
13960	{
13961	/* EFER MSR writes are always intercepted. */
13962	if (idMsr != MSR_K6_EFER)
13963	{
13964	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
13965	idMsr));
13966	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
13967	}
13968	}
13969
13970	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13971	{
13972	Assert(pVmcsInfo->pvMsrBitmap);
13973	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13974	if (fMsrpm & VMXMSRPM_ALLOW_WR)
13975	{
13976	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
13977	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
13978	}
13979	}
13980	break;
13981	}
13982	}
13983	}
13984	#endif /* VBOX_STRICT */
13985	}
13986	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13987	{
13988	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13989	rcStrict = VINF_SUCCESS;
13990	}
13991	else
13992	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13993
13994	return rcStrict;
13995	}
13996
13997
13998	/**
13999	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
14000	*/
14001	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14002	{
14003	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14004
14005	/** @todo The guest has likely hit a contended spinlock. We might want to
14006	* poke a schedule different guest VCPU. */
14007	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14008	if (RT_SUCCESS(rc))
14009	return VINF_EM_RAW_INTERRUPT;
14010
14011	AssertMsgFailed(("hmR0VmxExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
14012	return rc;
14013	}
14014
14015
14016	/**
14017	* VM-exit handler for when the TPR value is lowered below the specified
14018	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
14019	*/
14020	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14021	{
14022	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14023	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
14024
14025	/*
14026	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
14027	* We'll re-evaluate pending interrupts and inject them before the next VM
14028	* entry so we can just continue execution here.
14029	*/
14030	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
14031	return VINF_SUCCESS;
14032	}
14033
14034
14035	/**
14036	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
14037	* VM-exit.
14038	*
14039	* @retval VINF_SUCCESS when guest execution can continue.
14040	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
14041	* @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
14042	* incompatible guest state for VMX execution (real-on-v86 case).
14043	*/
14044	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14045	{
14046	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14047	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
14048
14049	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14050	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14051	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14052	AssertRCReturn(rc, rc);
14053
14054	VBOXSTRICTRC rcStrict;
14055	PVM pVM = pVCpu->CTX_SUFF(pVM);
14056	uint64_t const uExitQual = pVmxTransient->uExitQual;
14057	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
14058	switch (uAccessType)
14059	{
14060	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
14061	{
14062	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14063	AssertRCReturn(rc, rc);
14064
14065	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
14066	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
14067	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
14068	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
14069
14070	/*
14071	* MOV to CR3 only cause a VM-exit when one or more of the following are true:
14072	* - When nested paging isn't used.
14073	* - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
14074	* - We are executing in the VM debug loop.
14075	*/
14076	Assert( iCrReg != 3
14077	\|\| !pVM->hm.s.fNestedPaging
14078	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
14079	\|\| pVCpu->hm.s.fUsingDebugLoop);
14080
14081	/* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
14082	Assert( iCrReg != 8
14083	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
14084
14085	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
14086	AssertMsg( rcStrict == VINF_SUCCESS
14087	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14088
14089	/*
14090	* This is a kludge for handling switches back to real mode when we try to use
14091	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
14092	* deal with special selector values, so we have to return to ring-3 and run
14093	* there till the selector values are V86 mode compatible.
14094	*
14095	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
14096	* latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
14097	* this function.
14098	*/
14099	if ( iCrReg == 0
14100	&& rcStrict == VINF_SUCCESS
14101	&& !pVM->hm.s.vmx.fUnrestrictedGuest
14102	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
14103	&& (uOldCr0 & X86_CR0_PE)
14104	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
14105	{
14106	/** @todo Check selectors rather than returning all the time. */
14107	Assert(!pVmxTransient->fIsNestedGuest);
14108	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
14109	rcStrict = VINF_EM_RESCHEDULE_REM;
14110	}
14111	break;
14112	}
14113
14114	case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
14115	{
14116	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
14117	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
14118
14119	/*
14120	* MOV from CR3 only cause a VM-exit when one or more of the following are true:
14121	* - When nested paging isn't used.
14122	* - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
14123	* - We are executing in the VM debug loop.
14124	*/
14125	Assert( iCrReg != 3
14126	\|\| !pVM->hm.s.fNestedPaging
14127	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
14128	\|\| pVCpu->hm.s.fUsingDebugLoop);
14129
14130	/* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
14131	Assert( iCrReg != 8
14132	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
14133
14134	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
14135	break;
14136	}
14137
14138	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
14139	{
14140	/*
14141	* CLTS (Clear Task-Switch Flag in CR0).
14142	*/
14143	rcStrict = hmR0VmxExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbInstr);
14144	break;
14145	}
14146
14147	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
14148	{
14149	/*
14150	* LMSW (Load Machine-Status Word into CR0).
14151	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
14152	*/
14153	RTGCPTR GCPtrEffDst;
14154	uint8_t const cbInstr = pVmxTransient->cbInstr;
14155	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
14156	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
14157	if (fMemOperand)
14158	{
14159	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
14160	AssertRCReturn(rc, rc);
14161	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
14162	}
14163	else
14164	GCPtrEffDst = NIL_RTGCPTR;
14165	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
14166	break;
14167	}
14168
14169	default:
14170	{
14171	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
14172	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
14173	}
14174	}
14175
14176	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
14177	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
14178	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
14179
14180	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
14181	NOREF(pVM);
14182	return rcStrict;
14183	}
14184
14185
14186	/**
14187	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
14188	* VM-exit.
14189	*/
14190	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14191	{
14192	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14193	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
14194
14195	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14196	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14197	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14198	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14199	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
14200	\| CPUMCTX_EXTRN_EFER);
14201	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
14202	AssertRCReturn(rc, rc);
14203
14204	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
14205	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
14206	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
14207	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
14208	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
14209	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
14210	bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
14211	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
14212
14213	/*
14214	* Update exit history to see if this exit can be optimized.
14215	*/
14216	VBOXSTRICTRC rcStrict;
14217	PCEMEXITREC pExitRec = NULL;
14218	if ( !fGstStepping
14219	&& !fDbgStepping)
14220	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14221	!fIOString
14222	? !fIOWrite
14223	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
14224	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
14225	: !fIOWrite
14226	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
14227	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
14228	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14229	if (!pExitRec)
14230	{
14231	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
14232	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
14233
14234	uint32_t const cbValue = s_aIOSizes[uIOSize];
14235	uint32_t const cbInstr = pVmxTransient->cbInstr;
14236	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
14237	PVM pVM = pVCpu->CTX_SUFF(pVM);
14238	if (fIOString)
14239	{
14240	/*
14241	* INS/OUTS - I/O String instruction.
14242	*
14243	* Use instruction-information if available, otherwise fall back on
14244	* interpreting the instruction.
14245	*/
14246	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14247	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
14248	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
14249	if (fInsOutsInfo)
14250	{
14251	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14252	AssertRCReturn(rc2, rc2);
14253	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
14254	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
14255	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
14256	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
14257	if (fIOWrite)
14258	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
14259	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
14260	else
14261	{
14262	/*
14263	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
14264	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
14265	* See Intel Instruction spec. for "INS".
14266	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
14267	*/
14268	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
14269	}
14270	}
14271	else
14272	rcStrict = IEMExecOne(pVCpu);
14273
14274	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14275	fUpdateRipAlready = true;
14276	}
14277	else
14278	{
14279	/*
14280	* IN/OUT - I/O instruction.
14281	*/
14282	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14283	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
14284	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
14285	if (fIOWrite)
14286	{
14287	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
14288	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
14289	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14290	&& !pCtx->eflags.Bits.u1TF)
14291	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
14292	}
14293	else
14294	{
14295	uint32_t u32Result = 0;
14296	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
14297	if (IOM_SUCCESS(rcStrict))
14298	{
14299	/* Save result of I/O IN instr. in AL/AX/EAX. */
14300	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
14301	}
14302	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14303	&& !pCtx->eflags.Bits.u1TF)
14304	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
14305	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
14306	}
14307	}
14308
14309	if (IOM_SUCCESS(rcStrict))
14310	{
14311	if (!fUpdateRipAlready)
14312	{
14313	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
14314	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14315	}
14316
14317	/*
14318	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
14319	* while booting Fedora 17 64-bit guest.
14320	*
14321	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
14322	*/
14323	if (fIOString)
14324	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
14325
14326	/*
14327	* If any I/O breakpoints are armed, we need to check if one triggered
14328	* and take appropriate action.
14329	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
14330	*/
14331	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
14332	AssertRCReturn(rc, rc);
14333
14334	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
14335	* execution engines about whether hyper BPs and such are pending. */
14336	uint32_t const uDr7 = pCtx->dr[7];
14337	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
14338	&& X86_DR7_ANY_RW_IO(uDr7)
14339	&& (pCtx->cr4 & X86_CR4_DE))
14340	\|\| DBGFBpIsHwIoArmed(pVM)))
14341	{
14342	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
14343
14344	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
14345	VMMRZCallRing3Disable(pVCpu);
14346	HM_DISABLE_PREEMPT(pVCpu);
14347
14348	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
14349
14350	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
14351	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
14352	{
14353	/* Raise #DB. */
14354	if (fIsGuestDbgActive)
14355	ASMSetDR6(pCtx->dr[6]);
14356	if (pCtx->dr[7] != uDr7)
14357	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
14358
14359	hmR0VmxSetPendingXcptDB(pVCpu);
14360	}
14361	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
14362	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
14363	else if ( rcStrict2 != VINF_SUCCESS
14364	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
14365	rcStrict = rcStrict2;
14366	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
14367
14368	HM_RESTORE_PREEMPT();
14369	VMMRZCallRing3Enable(pVCpu);
14370	}
14371	}
14372
14373	#ifdef VBOX_STRICT
14374	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14375	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
14376	Assert(!fIOWrite);
14377	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14378	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
14379	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
14380	Assert(fIOWrite);
14381	else
14382	{
14383	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
14384	* statuses, that the VMM device and some others may return. See
14385	* IOM_SUCCESS() for guidance. */
14386	AssertMsg( RT_FAILURE(rcStrict)
14387	\|\| rcStrict == VINF_SUCCESS
14388	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
14389	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
14390	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
14391	\|\| rcStrict == VINF_EM_RAW_TO_R3
14392	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14393	# endif
14394	}
14395	#endif
14396	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
14397	}
14398	else
14399	{
14400	/*
14401	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14402	*/
14403	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14404	AssertRCReturn(rc2, rc2);
14405	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
14406	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
14407	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
14408	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14409	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
14410	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
14411
14412	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14413	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14414
14415	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14416	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14417	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14418	}
14419	return rcStrict;
14420	}
14421
14422
14423	/**
14424	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
14425	* VM-exit.
14426	*/
14427	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14428	{
14429	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14430
14431	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
14432	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14433	AssertRCReturn(rc, rc);
14434	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
14435	{
14436	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14437	AssertRCReturn(rc, rc);
14438	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
14439	{
14440	uint32_t uErrCode;
14441	RTGCUINTPTR GCPtrFaultAddress;
14442	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
14443	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
14444	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
14445	if (fErrorCodeValid)
14446	{
14447	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14448	AssertRCReturn(rc, rc);
14449	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
14450	}
14451	else
14452	uErrCode = 0;
14453
14454	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
14455	&& uVector == X86_XCPT_PF)
14456	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
14457	else
14458	GCPtrFaultAddress = 0;
14459
14460	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14461	AssertRCReturn(rc, rc);
14462
14463	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
14464	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
14465
14466	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
14467	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14468	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14469	}
14470	}
14471
14472	/* Fall back to the interpreter to emulate the task-switch. */
14473	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14474	return VERR_EM_INTERPRETER;
14475	}
14476
14477
14478	/**
14479	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
14480	*/
14481	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14482	{
14483	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14484
14485	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14486	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
14487	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14488	AssertRCReturn(rc, rc);
14489	return VINF_EM_DBG_STEPPED;
14490	}
14491
14492
14493	/**
14494	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
14495	*/
14496	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14497	{
14498	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14499	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
14500
14501	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14502	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14503	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14504	{
14505	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
14506	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14507	{
14508	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14509	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14510	}
14511	}
14512	else
14513	{
14514	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14515	rcStrict1 = VINF_SUCCESS;
14516	return rcStrict1;
14517	}
14518
14519	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
14520	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14521	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14522	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14523	AssertRCReturn(rc, rc);
14524
14525	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
14526	uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
14527	VBOXSTRICTRC rcStrict2;
14528	switch (uAccessType)
14529	{
14530	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
14531	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
14532	{
14533	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
14534	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
14535	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
14536
14537	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
14538	GCPhys &= PAGE_BASE_GC_MASK;
14539	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
14540	PVM pVM = pVCpu->CTX_SUFF(pVM);
14541	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
14542	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
14543
14544	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14545	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
14546	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
14547	CPUMCTX2CORE(pCtx), GCPhys);
14548	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14549	if ( rcStrict2 == VINF_SUCCESS
14550	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14551	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14552	{
14553	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14554	\| HM_CHANGED_GUEST_APIC_TPR);
14555	rcStrict2 = VINF_SUCCESS;
14556	}
14557	break;
14558	}
14559
14560	default:
14561	Log4Func(("uAccessType=%#x\n", uAccessType));
14562	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
14563	break;
14564	}
14565
14566	if (rcStrict2 != VINF_SUCCESS)
14567	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
14568	return rcStrict2;
14569	}
14570
14571
14572	/**
14573	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
14574	* VM-exit.
14575	*/
14576	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14577	{
14578	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14579
14580	/* We should -not- get this VM-exit if the guest's debug registers were active. */
14581	if (pVmxTransient->fWasGuestDebugStateActive)
14582	{
14583	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
14584	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14585	}
14586
14587	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14588	if ( !pVCpu->hm.s.fSingleInstruction
14589	&& !pVmxTransient->fWasHyperDebugStateActive)
14590	{
14591	Assert(!DBGFIsStepping(pVCpu));
14592	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
14593
14594	/* Don't intercept MOV DRx any more. */
14595	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
14596	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14597	AssertRCReturn(rc, rc);
14598
14599	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
14600	VMMRZCallRing3Disable(pVCpu);
14601	HM_DISABLE_PREEMPT(pVCpu);
14602
14603	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
14604	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
14605	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
14606
14607	HM_RESTORE_PREEMPT();
14608	VMMRZCallRing3Enable(pVCpu);
14609
14610	#ifdef VBOX_WITH_STATISTICS
14611	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14612	AssertRCReturn(rc, rc);
14613	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14614	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14615	else
14616	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14617	#endif
14618	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
14619	return VINF_SUCCESS;
14620	}
14621
14622	/*
14623	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
14624	* The EFER MSR is always up-to-date.
14625	* Update the segment registers and DR7 from the CPU.
14626	*/
14627	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14628	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14629	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
14630	AssertRCReturn(rc, rc);
14631	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
14632
14633	PVM pVM = pVCpu->CTX_SUFF(pVM);
14634	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14635	{
14636	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14637	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
14638	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
14639	if (RT_SUCCESS(rc))
14640	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
14641	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14642	}
14643	else
14644	{
14645	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14646	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
14647	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
14648	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14649	}
14650
14651	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
14652	if (RT_SUCCESS(rc))
14653	{
14654	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14655	AssertRCReturn(rc2, rc2);
14656	return VINF_SUCCESS;
14657	}
14658	return rc;
14659	}
14660
14661
14662	/**
14663	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
14664	* Conditional VM-exit.
14665	*/
14666	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14667	{
14668	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14669	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14670
14671	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14672	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14673	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14674	{
14675	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
14676	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
14677	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14678	{
14679	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14680	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14681	}
14682	}
14683	else
14684	{
14685	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14686	rcStrict1 = VINF_SUCCESS;
14687	return rcStrict1;
14688	}
14689
14690	/*
14691	* Get sufficent state and update the exit history entry.
14692	*/
14693	RTGCPHYS GCPhys;
14694	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14695	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14696	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14697	AssertRCReturn(rc, rc);
14698
14699	VBOXSTRICTRC rcStrict;
14700	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14701	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
14702	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14703	if (!pExitRec)
14704	{
14705	/*
14706	* If we succeed, resume guest execution.
14707	* If we fail in interpreting the instruction because we couldn't get the guest physical address
14708	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
14709	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
14710	* weird case. See @bugref{6043}.
14711	*/
14712	PVM pVM = pVCpu->CTX_SUFF(pVM);
14713	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14714	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
14715	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
14716	if ( rcStrict == VINF_SUCCESS
14717	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
14718	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
14719	{
14720	/* Successfully handled MMIO operation. */
14721	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14722	\| HM_CHANGED_GUEST_APIC_TPR);
14723	rcStrict = VINF_SUCCESS;
14724	}
14725	}
14726	else
14727	{
14728	/*
14729	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14730	*/
14731	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14732	AssertRCReturn(rc2, rc2);
14733
14734	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
14735	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
14736
14737	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14738	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14739
14740	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14741	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14742	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14743	}
14744	return VBOXSTRICTRC_TODO(rcStrict);
14745	}
14746
14747
14748	/**
14749	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
14750	* VM-exit.
14751	*/
14752	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14753	{
14754	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14755	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14756
14757	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14758	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14759	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14760	{
14761	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
14762	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14763	Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
14764	}
14765	else
14766	{
14767	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14768	rcStrict1 = VINF_SUCCESS;
14769	return rcStrict1;
14770	}
14771
14772	RTGCPHYS GCPhys;
14773	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14774	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14775	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14776	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14777	AssertRCReturn(rc, rc);
14778
14779	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
14780	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
14781
14782	RTGCUINT uErrorCode = 0;
14783	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
14784	uErrorCode \|= X86_TRAP_PF_ID;
14785	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
14786	uErrorCode \|= X86_TRAP_PF_RW;
14787	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
14788	uErrorCode \|= X86_TRAP_PF_P;
14789
14790	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
14791
14792
14793	/* Handle the pagefault trap for the nested shadow table. */
14794	PVM pVM = pVCpu->CTX_SUFF(pVM);
14795	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14796
14797	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
14798	pCtx->cs.Sel, pCtx->rip));
14799
14800	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
14801	TRPMResetTrap(pVCpu);
14802
14803	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
14804	if ( rcStrict2 == VINF_SUCCESS
14805	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14806	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14807	{
14808	/* Successfully synced our nested page tables. */
14809	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
14810	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
14811	return VINF_SUCCESS;
14812	}
14813
14814	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14815	return rcStrict2;
14816	}
14817
14818	/** @} */
14819
14820	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14821	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14822	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14823
14824	/**
14825	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
14826	*/
14827	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14828	{
14829	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14830	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
14831
14832	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
14833	AssertRCReturn(rc, rc);
14834
14835	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
14836	{
14837	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
14838	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
14839
14840	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
14841	* provides VM-exit instruction length. If this causes problem later,
14842	* disassemble the instruction like it's done on AMD-V. */
14843	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14844	AssertRCReturn(rc2, rc2);
14845	return rc;
14846	}
14847
14848	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14849	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14850	return rc;
14851	}
14852
14853
14854	/**
14855	* VM-exit exception handler for \#BP (Breakpoint exception).
14856	*/
14857	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14858	{
14859	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14860	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
14861
14862	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14863	AssertRCReturn(rc, rc);
14864
14865	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14866	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14867	if (rc == VINF_EM_RAW_GUEST_TRAP)
14868	{
14869	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14870	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14871	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14872	AssertRCReturn(rc, rc);
14873
14874	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14875	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14876	}
14877
14878	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
14879	return rc;
14880	}
14881
14882
14883	/**
14884	* VM-exit exception handler for \#AC (alignment check exception).
14885	*/
14886	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14887	{
14888	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14889
14890	/*
14891	* Re-inject it. We'll detect any nesting before getting here.
14892	*/
14893	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14894	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14895	AssertRCReturn(rc, rc);
14896	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
14897
14898	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14899	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14900	return VINF_SUCCESS;
14901	}
14902
14903
14904	/**
14905	* VM-exit exception handler for \#DB (Debug exception).
14906	*/
14907	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14908	{
14909	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14910	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
14911
14912	/*
14913	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
14914	* for processing.
14915	*/
14916	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14917
14918	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
14919	uint64_t const uDR6 = X86_DR6_INIT_VAL
14920	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
14921	\| X86_DR6_BD \| X86_DR6_BS));
14922
14923	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14924	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
14925	Log6Func(("rc=%Rrc\n", rc));
14926	if (rc == VINF_EM_RAW_GUEST_TRAP)
14927	{
14928	/*
14929	* The exception was for the guest. Update DR6, DR7.GD and
14930	* IA32_DEBUGCTL.LBR before forwarding it.
14931	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
14932	*/
14933	VMMRZCallRing3Disable(pVCpu);
14934	HM_DISABLE_PREEMPT(pVCpu);
14935
14936	pCtx->dr[6] &= ~X86_DR6_B_MASK;
14937	pCtx->dr[6] \|= uDR6;
14938	if (CPUMIsGuestDebugStateActive(pVCpu))
14939	ASMSetDR6(pCtx->dr[6]);
14940
14941	HM_RESTORE_PREEMPT();
14942	VMMRZCallRing3Enable(pVCpu);
14943
14944	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
14945	AssertRCReturn(rc, rc);
14946
14947	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
14948	pCtx->dr[7] &= ~X86_DR7_GD;
14949
14950	/* Paranoia. */
14951	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
14952	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
14953
14954	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
14955	AssertRCReturn(rc, rc);
14956
14957	/*
14958	* Raise #DB in the guest.
14959	*
14960	* It is important to reflect exactly what the VM-exit gave us (preserving the
14961	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
14962	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
14963	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
14964	*
14965	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
14966	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
14967	*/
14968	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14969	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14970	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14971	AssertRCReturn(rc, rc);
14972	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14973	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14974	return VINF_SUCCESS;
14975	}
14976
14977	/*
14978	* Not a guest trap, must be a hypervisor related debug event then.
14979	* Update DR6 in case someone is interested in it.
14980	*/
14981	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
14982	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
14983	CPUMSetHyperDR6(pVCpu, uDR6);
14984
14985	return rc;
14986	}
14987
14988
14989	/**
14990	* Hacks its way around the lovely mesa driver's backdoor accesses.
14991	*
14992	* @sa hmR0SvmHandleMesaDrvGp.
14993	*/
14994	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14995	{
14996	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
14997	RT_NOREF(pCtx);
14998
14999	/* For now we'll just skip the instruction. */
15000	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15001	}
15002
15003
15004	/**
15005	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
15006	* backdoor logging w/o checking what it is running inside.
15007	*
15008	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
15009	* backdoor port and magic numbers loaded in registers.
15010	*
15011	* @returns true if it is, false if it isn't.
15012	* @sa hmR0SvmIsMesaDrvGp.
15013	*/
15014	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
15015	{
15016	/* 0xed: IN eAX,dx */
15017	uint8_t abInstr[1];
15018	if (pVmxTransient->cbInstr != sizeof(abInstr))
15019	return false;
15020
15021	/* Check that it is #GP(0). */
15022	if (pVmxTransient->uExitIntErrorCode != 0)
15023	return false;
15024
15025	/* Check magic and port. */
15026	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
15027	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
15028	if (pCtx->rax != UINT32_C(0x564d5868))
15029	return false;
15030	if (pCtx->dx != UINT32_C(0x5658))
15031	return false;
15032
15033	/* Flat ring-3 CS. */
15034	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
15035	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
15036	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
15037	if (pCtx->cs.Attr.n.u2Dpl != 3)
15038	return false;
15039	if (pCtx->cs.u64Base != 0)
15040	return false;
15041
15042	/* Check opcode. */
15043	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
15044	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
15045	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
15046	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
15047	if (RT_FAILURE(rc))
15048	return false;
15049	if (abInstr[0] != 0xed)
15050	return false;
15051
15052	return true;
15053	}
15054
15055
15056	/**
15057	* VM-exit exception handler for \#GP (General-protection exception).
15058	*
15059	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
15060	*/
15061	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15062	{
15063	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15064	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
15065
15066	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15067	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15068	if (pVmcsInfo->RealMode.fRealOnV86Active)
15069	{ /* likely */ }
15070	else
15071	{
15072	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
15073	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv);
15074	#endif
15075	/* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
15076	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15077	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15078	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15079	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15080	AssertRCReturn(rc, rc);
15081	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
15082	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
15083
15084	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
15085	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
15086	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
15087	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15088	else
15089	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
15090	return rc;
15091	}
15092
15093	Assert(CPUMIsGuestInRealModeEx(pCtx));
15094	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
15095
15096	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15097	AssertRCReturn(rc, rc);
15098
15099	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
15100	if (rcStrict == VINF_SUCCESS)
15101	{
15102	if (!CPUMIsGuestInRealModeEx(pCtx))
15103	{
15104	/*
15105	* The guest is no longer in real-mode, check if we can continue executing the
15106	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
15107	*/
15108	pVmcsInfo->RealMode.fRealOnV86Active = false;
15109	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
15110	{
15111	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
15112	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15113	}
15114	else
15115	{
15116	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
15117	rcStrict = VINF_EM_RESCHEDULE;
15118	}
15119	}
15120	else
15121	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15122	}
15123	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15124	{
15125	rcStrict = VINF_SUCCESS;
15126	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15127	}
15128	return VBOXSTRICTRC_VAL(rcStrict);
15129	}
15130
15131
15132	/**
15133	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
15134	* the exception reported in the VMX transient structure back into the VM.
15135	*
15136	* @remarks Requires uExitIntInfo in the VMX transient structure to be
15137	* up-to-date.
15138	*/
15139	static VBOXSTRICTRC hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15140	{
15141	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15142	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
15143	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15144	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active,
15145	("uVector=%#x u32XcptBitmap=%#X32\n",
15146	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
15147	NOREF(pVmcsInfo);
15148	#endif
15149
15150	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
15151	hmR0VmxCheckExitDueToEventDelivery(). */
15152	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15153	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15154	AssertRCReturn(rc, rc);
15155	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15156
15157	#ifdef DEBUG_ramshankar
15158	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
15159	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n",
15160	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pCtx->cs.Sel, pCtx->rip));
15161	#endif
15162
15163	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15164	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15165	return VINF_SUCCESS;
15166	}
15167
15168
15169	/**
15170	* VM-exit exception handler for \#PF (Page-fault exception).
15171	*/
15172	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15173	{
15174	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15175	PVM pVM = pVCpu->CTX_SUFF(pVM);
15176	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15177	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15178	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15179	AssertRCReturn(rc, rc);
15180
15181	if (!pVM->hm.s.fNestedPaging)
15182	{ /* likely */ }
15183	else
15184	{
15185	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
15186	Assert(pVCpu->hm.s.fUsingDebugLoop);
15187	#endif
15188	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
15189	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
15190	{
15191	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15192	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
15193	}
15194	else
15195	{
15196	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15197	hmR0VmxSetPendingXcptDF(pVCpu);
15198	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
15199	}
15200	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15201	return rc;
15202	}
15203
15204	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
15205	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
15206	if (pVmxTransient->fVectoringPF)
15207	{
15208	Assert(pVCpu->hm.s.Event.fPending);
15209	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15210	}
15211
15212	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15213	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15214	AssertRCReturn(rc, rc);
15215
15216	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
15217	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
15218
15219	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
15220	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
15221
15222	Log4Func(("#PF: rc=%Rrc\n", rc));
15223	if (rc == VINF_SUCCESS)
15224	{
15225	/*
15226	* This is typically a shadow page table sync or a MMIO instruction. But we may have
15227	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
15228	*/
15229	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15230	TRPMResetTrap(pVCpu);
15231	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
15232	return rc;
15233	}
15234
15235	if (rc == VINF_EM_RAW_GUEST_TRAP)
15236	{
15237	if (!pVmxTransient->fVectoringDoublePF)
15238	{
15239	/* It's a guest page fault and needs to be reflected to the guest. */
15240	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
15241	TRPMResetTrap(pVCpu);
15242	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
15243	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15244	uGstErrorCode, pVmxTransient->uExitQual);
15245	}
15246	else
15247	{
15248	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15249	TRPMResetTrap(pVCpu);
15250	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
15251	hmR0VmxSetPendingXcptDF(pVCpu);
15252	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
15253	}
15254
15255	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15256	return VINF_SUCCESS;
15257	}
15258
15259	TRPMResetTrap(pVCpu);
15260	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
15261	return rc;
15262	}
15263
15264
15265	/**
15266	* VM-exit helper for LMSW.
15267	*/
15268	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw,
15269	RTGCPTR GCPtrEffDst)
15270	{
15271	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15272	AssertRCReturn(rc, rc);
15273
15274	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
15275	AssertMsg( rcStrict == VINF_SUCCESS
15276	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15277
15278	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
15279	if (rcStrict == VINF_IEM_RAISED_XCPT)
15280	{
15281	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15282	rcStrict = VINF_SUCCESS;
15283	}
15284
15285	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
15286	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15287	return rcStrict;
15288	}
15289
15290
15291	/**
15292	* VM-exit helper for CLTS.
15293	*/
15294	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
15295	{
15296	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15297	AssertRCReturn(rc, rc);
15298
15299	VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
15300	AssertMsg( rcStrict == VINF_SUCCESS
15301	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15302
15303	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
15304	if (rcStrict == VINF_IEM_RAISED_XCPT)
15305	{
15306	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15307	rcStrict = VINF_SUCCESS;
15308	}
15309
15310	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
15311	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15312	return rcStrict;
15313	}
15314
15315
15316	/**
15317	* VM-exit helper for MOV from CRx (CRx read).
15318	*/
15319	static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
15320	{
15321	Assert(iCrReg < 16);
15322	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
15323
15324	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15325	AssertRCReturn(rc, rc);
15326
15327	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
15328	AssertMsg( rcStrict == VINF_SUCCESS
15329	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15330
15331	if (iGReg == X86_GREG_xSP)
15332	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
15333	else
15334	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15335	#ifdef VBOX_WITH_STATISTICS
15336	switch (iCrReg)
15337	{
15338	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
15339	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
15340	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
15341	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
15342	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
15343	}
15344	#endif
15345	Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
15346	return rcStrict;
15347	}
15348
15349
15350	/**
15351	* VM-exit helper for MOV to CRx (CRx write).
15352	*/
15353	static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
15354	{
15355	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15356	AssertRCReturn(rc, rc);
15357
15358	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
15359	AssertMsg( rcStrict == VINF_SUCCESS
15360	\|\| rcStrict == VINF_IEM_RAISED_XCPT
15361	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15362
15363	switch (iCrReg)
15364	{
15365	case 0:
15366	{
15367	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
15368	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
15369	Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
15370	break;
15371	}
15372
15373	case 2:
15374	{
15375	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
15376	/* Nothing to do here, CR2 it's not part of the VMCS. */
15377	break;
15378	}
15379
15380	case 3:
15381	{
15382	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
15383	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
15384	Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
15385	break;
15386	}
15387
15388	case 4:
15389	{
15390	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
15391	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
15392	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
15393	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
15394	break;
15395	}
15396
15397	case 8:
15398	{
15399	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
15400	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
15401	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
15402	break;
15403	}
15404
15405	default:
15406	AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
15407	break;
15408	}
15409
15410	if (rcStrict == VINF_IEM_RAISED_XCPT)
15411	{
15412	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15413	rcStrict = VINF_SUCCESS;
15414	}
15415	return rcStrict;
15416	}
15417
15418
15419	/**
15420	* VM-exit helper for handling host NMIs.
15421	*/
15422	static VBOXSTRICTRC hmR0VmxExitHostNmi(PVMCPU pVCpu)
15423	{
15424	VMXDispatchHostNmi();
15425
15426	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
15427	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
15428	return VINF_SUCCESS;
15429	}
15430
15431
15432	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15433	/** @name VMX instruction handlers.
15434	* @{
15435	*/
15436	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15437	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VMX instructions VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15438	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15439
15440	/**
15441	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
15442	*/
15443	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15444	{
15445	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15446
15447	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15448	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15449	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15450	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15451	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15452	AssertRCReturn(rc, rc);
15453
15454	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15455
15456	VMXVEXITINFO ExitInfo;
15457	RT_ZERO(ExitInfo);
15458	ExitInfo.uReason = pVmxTransient->uExitReason;
15459	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15460	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15461	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15462	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15463
15464	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
15465	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15466	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15467	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15468	{
15469	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15470	rcStrict = VINF_SUCCESS;
15471	}
15472	return rcStrict;
15473	}
15474
15475
15476	/**
15477	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
15478	*/
15479	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15480	{
15481	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15482
15483	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
15484	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15485	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15486	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15487	AssertRCReturn(rc, rc);
15488
15489	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15490
15491	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
15492	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15493	{
15494	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15495	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15496	}
15497	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15498	return rcStrict;
15499	}
15500
15501
15502	/**
15503	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
15504	*/
15505	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15506	{
15507	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15508
15509	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15510	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15511	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15512	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15513	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15514	AssertRCReturn(rc, rc);
15515
15516	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15517
15518	VMXVEXITINFO ExitInfo;
15519	RT_ZERO(ExitInfo);
15520	ExitInfo.uReason = pVmxTransient->uExitReason;
15521	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15522	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15523	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15524	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15525
15526	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
15527	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15528	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15529	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15530	{
15531	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15532	rcStrict = VINF_SUCCESS;
15533	}
15534	return rcStrict;
15535	}
15536
15537
15538	/**
15539	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
15540	*/
15541	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15542	{
15543	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15544
15545	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15546	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15547	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15548	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15549	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15550	AssertRCReturn(rc, rc);
15551
15552	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15553
15554	VMXVEXITINFO ExitInfo;
15555	RT_ZERO(ExitInfo);
15556	ExitInfo.uReason = pVmxTransient->uExitReason;
15557	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15558	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15559	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15560	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15561
15562	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
15563	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15564	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15565	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15566	{
15567	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15568	rcStrict = VINF_SUCCESS;
15569	}
15570	return rcStrict;
15571	}
15572
15573
15574	/**
15575	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
15576	*/
15577	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15578	{
15579	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15580
15581	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15582	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15583	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15584	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15585	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15586	AssertRCReturn(rc, rc);
15587
15588	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15589
15590	VMXVEXITINFO ExitInfo;
15591	RT_ZERO(ExitInfo);
15592	ExitInfo.uReason = pVmxTransient->uExitReason;
15593	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15594	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15595	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15596	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15597	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15598
15599	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
15600	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15601	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15602	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15603	{
15604	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15605	rcStrict = VINF_SUCCESS;
15606	}
15607	return rcStrict;
15608	}
15609
15610
15611	/**
15612	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
15613	*/
15614	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15615	{
15616	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15617
15618	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
15619	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15620	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15621	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15622	AssertRCReturn(rc, rc);
15623
15624	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15625
15626	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
15627	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15628	{
15629	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15630	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15631	}
15632	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15633	return rcStrict;
15634	}
15635
15636
15637	/**
15638	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
15639	*/
15640	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15641	{
15642	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15643
15644	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15645	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15646	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15647	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15648	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15649	AssertRCReturn(rc, rc);
15650
15651	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15652
15653	VMXVEXITINFO ExitInfo;
15654	RT_ZERO(ExitInfo);
15655	ExitInfo.uReason = pVmxTransient->uExitReason;
15656	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15657	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15658	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15659	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15660	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15661
15662	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
15663	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15664	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15665	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15666	{
15667	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15668	rcStrict = VINF_SUCCESS;
15669	}
15670	return rcStrict;
15671	}
15672
15673
15674	/**
15675	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
15676	*/
15677	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15678	{
15679	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15680
15681	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15682	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
15683	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15684	AssertRCReturn(rc, rc);
15685
15686	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15687
15688	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
15689	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15690	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
15691	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15692	{
15693	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15694	rcStrict = VINF_SUCCESS;
15695	}
15696	return rcStrict;
15697	}
15698
15699
15700	/**
15701	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
15702	*/
15703	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15704	{
15705	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15706
15707	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15708	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15709	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15710	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15711	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15712	AssertRCReturn(rc, rc);
15713
15714	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15715
15716	VMXVEXITINFO ExitInfo;
15717	RT_ZERO(ExitInfo);
15718	ExitInfo.uReason = pVmxTransient->uExitReason;
15719	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15720	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15721	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15722	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15723
15724	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
15725	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15726	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15727	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15728	{
15729	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15730	rcStrict = VINF_SUCCESS;
15731	}
15732	return rcStrict;
15733	}
15734
15735
15736	/**
15737	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
15738	*/
15739	HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15740	{
15741	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15742
15743	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15744	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15745	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15746	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15747	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15748	AssertRCReturn(rc, rc);
15749
15750	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15751
15752	VMXVEXITINFO ExitInfo;
15753	RT_ZERO(ExitInfo);
15754	ExitInfo.uReason = pVmxTransient->uExitReason;
15755	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15756	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15757	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15758	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15759
15760	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
15761	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15762	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15763	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15764	{
15765	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15766	rcStrict = VINF_SUCCESS;
15767	}
15768	return rcStrict;
15769	}
15770
15771	/** @} */
15772
15773	/** @name Nested-guest VM-exit handlers.
15774	* @{
15775	*/
15776	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15777	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15778	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15779
15780
15781	/**
15782	* Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
15783	* Unconditional VM-exit.
15784	*/
15785	HMVMX_EXIT_DECL hmR0VmxExitTripleFaultNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15786	{
15787	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15788	return IEMExecVmxVmexitTripleFault(pVCpu);
15789	}
15790
15791
15792	/**
15793	* Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
15794	*/
15795	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15796	{
15797	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15798
15799	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
15800	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
15801	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
15802	}
15803
15804
15805	/**
15806	* Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
15807	*/
15808	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15809	{
15810	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15811
15812	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
15813	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
15814	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
15815	}
15816
15817
15818	/**
15819	* Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
15820	* Unconditional VM-exit.
15821	*/
15822	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitchNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15823	{
15824	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15825
15826	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15827	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15828	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15829	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15830	AssertRCReturn(rc, rc);
15831
15832	VMXVEXITINFO ExitInfo;
15833	RT_ZERO(ExitInfo);
15834	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15835	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15836
15837	VMXVEXITEVENTINFO ExitEventInfo;
15838	RT_ZERO(ExitInfo);
15839	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
15840	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
15841	return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
15842	}
15843
15844
15845	/**
15846	* Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
15847	*/
15848	HMVMX_EXIT_DECL hmR0VmxExitHltNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15849	{
15850	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15851
15852	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
15853	{
15854	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15855	AssertRCReturn(rc, rc);
15856	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
15857	}
15858	return hmR0VmxExitHlt(pVCpu, pVmxTransient);
15859	}
15860
15861
15862	/**
15863	* Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
15864	*/
15865	HMVMX_EXIT_DECL hmR0VmxExitInvlpgNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15866	{
15867	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15868
15869	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
15870	{
15871	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15872	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15873	AssertRCReturn(rc, rc);
15874
15875	VMXVEXITINFO ExitInfo;
15876	RT_ZERO(ExitInfo);
15877	ExitInfo.uReason = pVmxTransient->uExitReason;
15878	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15879	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15880	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
15881	}
15882	return hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
15883	}
15884
15885
15886	/**
15887	* Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
15888	*/
15889	HMVMX_EXIT_DECL hmR0VmxExitRdpmcNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15890	{
15891	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15892
15893	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
15894	{
15895	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15896	AssertRCReturn(rc, rc);
15897	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
15898	}
15899	return hmR0VmxExitRdpmc(pVCpu, pVmxTransient);
15900	}
15901
15902
15903	/**
15904	* Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
15905	* (VMX_EXIT_VMWRITE). Conditional VM-exit.
15906	*/
15907	HMVMX_EXIT_DECL hmR0VmxExitVmreadVmwriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15908	{
15909	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15910
15911	Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD
15912	\|\| pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
15913
15914	int rc = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15915	AssertRCReturn(rc, rc);
15916
15917	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
15918	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
15919	uint64_t u64FieldEnc = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
15920
15921	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
15922	if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
15923	u64FieldEnc &= UINT64_C(0xffffffff);
15924
15925	if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64FieldEnc))
15926	{
15927	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15928	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15929	AssertRCReturn(rc, rc);
15930
15931	VMXVEXITINFO ExitInfo;
15932	RT_ZERO(ExitInfo);
15933	ExitInfo.uReason = pVmxTransient->uExitReason;
15934	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15935	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15936	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
15937	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
15938	}
15939
15940	if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
15941	return hmR0VmxExitVmread(pVCpu, pVmxTransient);
15942	return hmR0VmxExitVmwrite(pVCpu, pVmxTransient);
15943	}
15944
15945
15946	/**
15947	* Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
15948	*/
15949	HMVMX_EXIT_DECL hmR0VmxExitRdtscNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15950	{
15951	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15952
15953	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
15954	{
15955	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15956	AssertRCReturn(rc, rc);
15957	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
15958	}
15959
15960	return hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
15961	}
15962
15963
15964	/**
15965	* Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
15966	* Conditional VM-exit.
15967	*/
15968	HMVMX_EXIT_DECL hmR0VmxExitMovCRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15969	{
15970	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15971
15972	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15973	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15974	AssertRCReturn(rc, rc);
15975
15976	VBOXSTRICTRC rcStrict;
15977	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
15978	switch (uAccessType)
15979	{
15980	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
15981	{
15982	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
15983	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
15984	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
15985	uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
15986	if (CPUMIsGuestVmxMovToCr0Cr4InterceptSet(pVCpu, &pVCpu->cpum.GstCtx, iCrReg, uNewCrX))
15987	{
15988	VMXVEXITINFO ExitInfo;
15989	RT_ZERO(ExitInfo);
15990	ExitInfo.uReason = pVmxTransient->uExitReason;
15991	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15992	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15993	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
15994	}
15995	else
15996	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
15997	break;
15998	}
15999
16000	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
16001	{
16002	/*
16003	* CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
16004	* CR2 reads do not cause a VM-exit.
16005	* CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
16006	* CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
16007	*/
16008	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16009	if ( iCrReg == 3
16010	\|\| iCrReg == 8)
16011	{
16012	static const uint32_t s_aCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
16013	0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
16014	uint32_t const uIntercept = s_aCrXReadIntercepts[iCrReg];
16015	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uIntercept))
16016	{
16017	VMXVEXITINFO ExitInfo;
16018	RT_ZERO(ExitInfo);
16019	ExitInfo.uReason = pVmxTransient->uExitReason;
16020	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16021	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16022	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16023	}
16024	else
16025	{
16026	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16027	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
16028	}
16029	}
16030	else
16031	{
16032	AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
16033	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
16034	}
16035	break;
16036	}
16037
16038	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
16039	{
16040	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
16041	Assert(pVmcsNstGst);
16042	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
16043	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
16044	if ( (uGstHostMask & X86_CR0_TS)
16045	&& (uReadShadow & X86_CR0_TS))
16046	{
16047	VMXVEXITINFO ExitInfo;
16048	RT_ZERO(ExitInfo);
16049	ExitInfo.uReason = pVmxTransient->uExitReason;
16050	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16051	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16052	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16053	}
16054	else
16055	rcStrict = hmR0VmxExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr);
16056	break;
16057	}
16058
16059	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
16060	{
16061	RTGCPTR GCPtrEffDst;
16062	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
16063	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
16064	if (fMemOperand)
16065	{
16066	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
16067	AssertRCReturn(rc, rc);
16068	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
16069	}
16070	else
16071	GCPtrEffDst = NIL_RTGCPTR;
16072
16073	if (CPUMIsGuestVmxLmswInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uNewMsw))
16074	{
16075	VMXVEXITINFO ExitInfo;
16076	RT_ZERO(ExitInfo);
16077	ExitInfo.uReason = pVmxTransient->uExitReason;
16078	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16079	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
16080	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16081	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16082	}
16083	else
16084	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, uNewMsw, GCPtrEffDst);
16085	break;
16086	}
16087
16088	default:
16089	{
16090	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
16091	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
16092	}
16093	}
16094
16095	if (rcStrict == VINF_IEM_RAISED_XCPT)
16096	{
16097	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16098	rcStrict = VINF_SUCCESS;
16099	}
16100	return rcStrict;
16101	}
16102
16103
16104	/**
16105	* Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
16106	* Conditional VM-exit.
16107	*/
16108	HMVMX_EXIT_DECL hmR0VmxExitMovDRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16109	{
16110	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16111
16112	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
16113	{
16114	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16115	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16116	AssertRCReturn(rc, rc);
16117
16118	VMXVEXITINFO ExitInfo;
16119	RT_ZERO(ExitInfo);
16120	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16121	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16122	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16123	}
16124	return hmR0VmxExitMovDRx(pVCpu, pVmxTransient);
16125	}
16126
16127
16128	/**
16129	* Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
16130	* Conditional VM-exit.
16131	*/
16132	HMVMX_EXIT_DECL hmR0VmxExitIoInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16133	{
16134	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16135
16136	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16137	AssertRCReturn(rc, rc);
16138
16139	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
16140	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
16141	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
16142
16143	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
16144	uint8_t const cbAccess = s_aIOSizes[uIOSize];
16145	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
16146	{
16147	/*
16148	* IN/OUT instruction:
16149	* - Provides VM-exit instruction length.
16150	*
16151	* INS/OUTS instruction:
16152	* - Provides VM-exit instruction length.
16153	* - Provides Guest-linear address.
16154	* - Optionally provides VM-exit instruction info (depends on CPU feature).
16155	*/
16156	PVM pVM = pVCpu->CTX_SUFF(pVM);
16157	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16158	AssertRCReturn(rc, rc);
16159
16160	/* Make sure we don't use stale/uninitialized VMX-transient info. below. */
16161	pVmxTransient->ExitInstrInfo.u = 0;
16162	pVmxTransient->uGuestLinearAddr = 0;
16163
16164	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
16165	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
16166	if (fIOString)
16167	{
16168	rc \|= hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
16169	if (fVmxInsOutsInfo)
16170	{
16171	Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
16172	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16173	}
16174	}
16175	AssertRCReturn(rc, rc);
16176
16177	VMXVEXITINFO ExitInfo;
16178	RT_ZERO(ExitInfo);
16179	ExitInfo.uReason = pVmxTransient->uExitReason;
16180	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16181	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16182	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16183	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
16184	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16185	}
16186	return hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
16187	}
16188
16189
16190	/**
16191	* Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
16192	*/
16193	HMVMX_EXIT_DECL hmR0VmxExitRdmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16194	{
16195	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16196
16197	uint32_t fMsrpm;
16198	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
16199	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
16200	else
16201	fMsrpm = VMXMSRPM_EXIT_RD;
16202
16203	if (fMsrpm & VMXMSRPM_EXIT_RD)
16204	{
16205	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16206	AssertRCReturn(rc, rc);
16207	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16208	}
16209	return hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
16210	}
16211
16212
16213	/**
16214	* Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
16215	*/
16216	HMVMX_EXIT_DECL hmR0VmxExitWrmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16217	{
16218	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16219
16220	uint32_t fMsrpm;
16221	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
16222	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
16223	else
16224	fMsrpm = VMXMSRPM_EXIT_WR;
16225
16226	if (fMsrpm & VMXMSRPM_EXIT_WR)
16227	{
16228	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16229	AssertRCReturn(rc, rc);
16230	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16231	}
16232	return hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
16233	}
16234
16235
16236	/**
16237	* Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
16238	*/
16239	HMVMX_EXIT_DECL hmR0VmxExitMwaitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16240	{
16241	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16242
16243	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
16244	{
16245	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16246	AssertRCReturn(rc, rc);
16247	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16248	}
16249	return hmR0VmxExitMwait(pVCpu, pVmxTransient);
16250	}
16251
16252
16253	/**
16254	* Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
16255	* VM-exit.
16256	*/
16257	HMVMX_EXIT_DECL hmR0VmxExitMtfNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16258	{
16259	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16260
16261	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
16262	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16263	}
16264
16265
16266	/**
16267	* Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
16268	*/
16269	HMVMX_EXIT_DECL hmR0VmxExitMonitorNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16270	{
16271	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16272
16273	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
16274	{
16275	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16276	AssertRCReturn(rc, rc);
16277	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16278	}
16279	return hmR0VmxExitMonitor(pVCpu, pVmxTransient);
16280	}
16281
16282
16283	/**
16284	* Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
16285	*/
16286	HMVMX_EXIT_DECL hmR0VmxExitPauseNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16287	{
16288	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16289
16290	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
16291	* PAUSE when executing a nested-guest? If it does not, we would not need
16292	* to check for the intercepts here. Just call VM-exit... */
16293
16294	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
16295	if ( CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
16296	\|\| CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
16297	{
16298	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16299	AssertRCReturn(rc, rc);
16300	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16301	}
16302	return hmR0VmxExitPause(pVCpu, pVmxTransient);
16303	}
16304
16305
16306	/**
16307	* Nested-guest VM-exit handler for when the TPR value is lowered below the
16308	* specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
16309	*/
16310	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThresholdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16311	{
16312	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16313
16314	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
16315	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16316	return hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient);
16317	}
16318
16319
16320	/**
16321	* Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
16322	* VM-exit.
16323	*/
16324	HMVMX_EXIT_DECL hmR0VmxExitApicAccessNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16325	{
16326	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16327
16328	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
16329	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16330	AssertRCReturn(rc, rc);
16331
16332	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
16333	}
16334
16335
16336	/**
16337	* Nested-guest VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
16338	* (VMX_EXIT_GDTR_IDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
16339	* Conditional VM-exit.
16340	*/
16341	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccessNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16342	{
16343	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16344
16345	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
16346	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16347	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16348	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16349	AssertRCReturn(rc, rc);
16350
16351	VMXVEXITINFO ExitInfo;
16352	RT_ZERO(ExitInfo);
16353	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16354	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16355	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16356	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16357	}
16358
16359
16360	/**
16361	* Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
16362	*/
16363	HMVMX_EXIT_DECL hmR0VmxExitRdtscpNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16364	{
16365	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16366
16367	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
16368	{
16369	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
16370	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16371	AssertRCReturn(rc, rc);
16372	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16373	}
16374	return hmR0VmxExitRdtscp(pVCpu, pVmxTransient);
16375	}
16376
16377
16378	/**
16379	* Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
16380	*/
16381	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16382	{
16383	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16384
16385	if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
16386	{
16387	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16388	AssertRCReturn(rc, rc);
16389	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16390	}
16391	return hmR0VmxExitWbinvd(pVCpu, pVmxTransient);
16392	}
16393
16394
16395	/**
16396	* Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
16397	*/
16398	HMVMX_EXIT_DECL hmR0VmxExitInvpcidNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16399	{
16400	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16401
16402	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
16403	{
16404	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
16405	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16406	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16407	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16408	AssertRCReturn(rc, rc);
16409
16410	VMXVEXITINFO ExitInfo;
16411	RT_ZERO(ExitInfo);
16412	ExitInfo.uReason = pVmxTransient->uExitReason;
16413	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16414	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16415	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16416	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16417	}
16418	return hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
16419	}
16420
16421
16422	/**
16423	* Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
16424	* and only provide the instruction length.
16425	*
16426	* Unconditional VM-exit.
16427	*/
16428	HMVMX_EXIT_DECL hmR0VmxExitInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16429	{
16430	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16431
16432	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16433	AssertRCReturn(rc, rc);
16434	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16435	}
16436
16437
16438	/**
16439	* Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
16440	* but provide instruction length as well as more information.
16441	*
16442	* Unconditional VM-exit.
16443	*/
16444	HMVMX_EXIT_DECL hmR0VmxExitInstrWithInfoNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16445	{
16446	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16447
16448	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16449	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16450	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16451	AssertRCReturn(rc, rc);
16452
16453	VMXVEXITINFO ExitInfo;
16454	RT_ZERO(ExitInfo);
16455	ExitInfo.uReason = pVmxTransient->uExitReason;
16456	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16457	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16458	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16459	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16460	}
16461
16462	/** @} */
16463
16464	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
16465

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 78927

Download in other formats: