HMVMXR0.cpp@ 78481

Last change on this file since 78481 was 78481, checked in by vboxsync, 6 years ago
VMM/HM, IEM: Nested VMX: bugref:9180 Hardware-assisted VMX VM-exit handling interface bits, I/O exit handling, comments and disabled code on what needs to be done with future optimizations.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 649.3 KB

Line
1	/* $Id: HMVMXR0.cpp 78481 2019-05-13 09:52:54Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27
28	#include <VBox/vmm/pdmapi.h>
29	#include <VBox/vmm/dbgf.h>
30	#include <VBox/vmm/iem.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#include <VBox/vmm/em.h>
35	#include <VBox/vmm/gim.h>
36	#include <VBox/vmm/apic.h>
37	#ifdef VBOX_WITH_REM
38	# include <VBox/vmm/rem.h>
39	#endif
40	#include "HMInternal.h"
41	#include <VBox/vmm/vm.h>
42	#include <VBox/vmm/hmvmxinline.h>
43	#include "HMVMXR0.h"
44	#include "dtrace/VBoxVMM.h"
45
46	#ifdef DEBUG_ramshankar
47	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
48	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
49	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
50	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
51	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
52	# define HMVMX_ALWAYS_TRAP_PF
53	# define HMVMX_ALWAYS_FLUSH_TLB
54	# define HMVMX_ALWAYS_SWAP_EFER
55	#endif
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	/** Use the function table. */
62	#define HMVMX_USE_FUNCTION_TABLE
63
64	/** Determine which tagged-TLB flush handler to use. */
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70	/** @name HMVMX_READ_XXX
71	* Flags to skip redundant reads of some common VMCS fields that are not part of
72	* the guest-CPU or VCPU state but are needed while handling VM-exits.
73	*/
74	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
75	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
76	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
77	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
78	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
79	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
80	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
81	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
82	/** @} */
83
84	/**
85	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
86	* guest using hardware-assisted VMX.
87	*
88	* This excludes state like GPRs (other than RSP) which are always are
89	* swapped and restored across the world-switch and also registers like EFER,
90	* MSR which cannot be modified by the guest without causing a VM-exit.
91	*/
92	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
93	\| CPUMCTX_EXTRN_RFLAGS \
94	\| CPUMCTX_EXTRN_RSP \
95	\| CPUMCTX_EXTRN_SREG_MASK \
96	\| CPUMCTX_EXTRN_TABLE_MASK \
97	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
98	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
99	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
100	\| CPUMCTX_EXTRN_TSC_AUX \
101	\| CPUMCTX_EXTRN_OTHER_MSRS \
102	\| CPUMCTX_EXTRN_CR0 \
103	\| CPUMCTX_EXTRN_CR3 \
104	\| CPUMCTX_EXTRN_CR4 \
105	\| CPUMCTX_EXTRN_DR7 \
106	\| CPUMCTX_EXTRN_HM_VMX_MASK)
107
108	/**
109	* Exception bitmap mask for real-mode guests (real-on-v86).
110	*
111	* We need to intercept all exceptions manually except:
112	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
113	* due to bugs in Intel CPUs.
114	* - \#PF need not be intercepted even in real-mode if we have nested paging
115	* support.
116	*/
117	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
118	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
119	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
120	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
121	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
122	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
123	\| RT_BIT(X86_XCPT_XF))
124
125	/** Maximum VM-instruction error number. */
126	#define HMVMX_INSTR_ERROR_MAX 28
127
128	/** Profiling macro. */
129	#ifdef HM_PROFILE_EXIT_DISPATCH
130	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
131	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
132	#else
133	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
134	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
135	#endif
136
137	/** Assert that preemption is disabled or covered by thread-context hooks. */
138	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
139	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
140
141	/** Assert that we haven't migrated CPUs when thread-context hooks are not
142	* used. */
143	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
144	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
145	("Illegal migration! Entered on CPU %u Current %u\n", \
146	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
147
148	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
149	* context. */
150	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
151	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
152	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
153
154	/** Helper macro for VM-exit handlers called unexpectedly. */
155	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
156	do { \
157	(a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
158	return VERR_VMX_UNEXPECTED_EXIT; \
159	} while (0)
160
161	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
162	/** Macro that does the necessary privilege checks and intercepted VM-exits for
163	* guests that attempted to execute a VMX instruction. */
164	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
165	do \
166	{ \
167	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
168	if (rcStrictTmp == VINF_SUCCESS) \
169	{ /* likely */ } \
170	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
171	{ \
172	Assert((a_pVCpu)->hm.s.Event.fPending); \
173	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
174	return VINF_SUCCESS; \
175	} \
176	else \
177	{ \
178	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
179	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
180	} \
181	} while (0)
182
183	/** Macro that decodes a memory operand for an instruction VM-exit. */
184	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
185	do \
186	{ \
187	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
188	(a_pGCPtrEffAddr)); \
189	if (rcStrictTmp == VINF_SUCCESS) \
190	{ /* likely */ } \
191	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
192	{ \
193	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
194	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
195	NOREF(uXcptTmp); \
196	return VINF_SUCCESS; \
197	} \
198	else \
199	{ \
200	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
201	return rcStrictTmp; \
202	} \
203	} while (0)
204
205	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
206
207
208	/*********************************************************************************************************************************
209	* Structures and Typedefs *
210	*********************************************************************************************************************************/
211	/**
212	* VMX transient state.
213	*
214	* A state structure for holding miscellaneous information across
215	* VMX non-root operation and restored after the transition.
216	*/
217	typedef struct VMXTRANSIENT
218	{
219	/** The host's rflags/eflags. */
220	RTCCUINTREG fEFlags;
221	#if HC_ARCH_BITS == 32
222	uint32_t u32Alignment0;
223	#endif
224	/** The guest's TPR value used for TPR shadowing. */
225	uint8_t u8GuestTpr;
226	/** Alignment. */
227	uint8_t abAlignment0[7];
228
229	/** The basic VM-exit reason. */
230	uint16_t uExitReason;
231	/** Alignment. */
232	uint16_t u16Alignment0;
233	/** The VM-exit interruption error code. */
234	uint32_t uExitIntErrorCode;
235	/** The VM-exit exit code qualification. */
236	uint64_t uExitQual;
237	/** The Guest-linear address. */
238	uint64_t uGuestLinearAddr;
239
240	/** The VM-exit interruption-information field. */
241	uint32_t uExitIntInfo;
242	/** The VM-exit instruction-length field. */
243	uint32_t cbInstr;
244	/** The VM-exit instruction-information field. */
245	VMXEXITINSTRINFO ExitInstrInfo;
246	/** Whether the VM-entry failed or not. */
247	bool fVMEntryFailed;
248	/** Whether we are currently executing a nested-guest. */
249	bool fIsNestedGuest;
250	/** Alignment. */
251	uint8_t abAlignment1[2];
252
253	/** The VM-entry interruption-information field. */
254	uint32_t uEntryIntInfo;
255	/** The VM-entry exception error code field. */
256	uint32_t uEntryXcptErrorCode;
257	/** The VM-entry instruction length field. */
258	uint32_t cbEntryInstr;
259
260	/** IDT-vectoring information field. */
261	uint32_t uIdtVectoringInfo;
262	/** IDT-vectoring error code. */
263	uint32_t uIdtVectoringErrorCode;
264
265	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
266	uint32_t fVmcsFieldsRead;
267
268	/** Whether the guest debug state was active at the time of VM-exit. */
269	bool fWasGuestDebugStateActive;
270	/** Whether the hyper debug state was active at the time of VM-exit. */
271	bool fWasHyperDebugStateActive;
272	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
273	bool fUpdatedTscOffsettingAndPreemptTimer;
274	/** Whether the VM-exit was caused by a page-fault during delivery of a
275	* contributory exception or a page-fault. */
276	bool fVectoringDoublePF;
277	/** Whether the VM-exit was caused by a page-fault during delivery of an
278	* external interrupt or NMI. */
279	bool fVectoringPF;
280	bool afAlignment0[3];
281
282	/** The VMCS info. object. */
283	PVMXVMCSINFO pVmcsInfo;
284	} VMXTRANSIENT;
285	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
286	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
287	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
288	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
289	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
290	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
291	/** Pointer to VMX transient state. */
292	typedef VMXTRANSIENT *PVMXTRANSIENT;
293
294	/**
295	* Memory operand read or write access.
296	*/
297	typedef enum VMXMEMACCESS
298	{
299	VMXMEMACCESS_READ = 0,
300	VMXMEMACCESS_WRITE = 1
301	} VMXMEMACCESS;
302
303	/**
304	* VMX VM-exit handler.
305	*
306	* @returns Strict VBox status code (i.e. informational status codes too).
307	* @param pVCpu The cross context virtual CPU structure.
308	* @param pVmxTransient The VMX-transient structure.
309	*/
310	#ifndef HMVMX_USE_FUNCTION_TABLE
311	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
312	#else
313	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
314	/** Pointer to VM-exit handler. */
315	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
316	#endif
317
318	/**
319	* VMX VM-exit handler, non-strict status code.
320	*
321	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
322	*
323	* @returns VBox status code, no informational status code returned.
324	* @param pVCpu The cross context virtual CPU structure.
325	* @param pVmxTransient The VMX-transient structure.
326	*
327	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
328	* use of that status code will be replaced with VINF_EM_SOMETHING
329	* later when switching over to IEM.
330	*/
331	#ifndef HMVMX_USE_FUNCTION_TABLE
332	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
333	#else
334	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
335	#endif
336
337
338	/*********************************************************************************************************************************
339	* Internal Functions *
340	*********************************************************************************************************************************/
341	#ifndef HMVMX_USE_FUNCTION_TABLE
342	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
343	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
344	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
345	#else
346	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
347	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
348	#endif
349	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
350	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
351	#endif
352
353	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
354	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
355	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
356	#endif
357
358	/** @name VM-exit handlers.
359	* @{
360	*/
361	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
362	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
363	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
364	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
365	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
366	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
367	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
368	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
369	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
370	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
371	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
372	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
373	static FNVMXEXITHANDLER hmR0VmxExitHlt;
374	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
375	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
376	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
377	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
378	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
379	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
380	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
381	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
382	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
383	static FNVMXEXITHANDLER hmR0VmxExitVmread;
384	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
385	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
386	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
387	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
388	#endif
389	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
390	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
391	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
392	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
393	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
394	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
395	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
396	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
397	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
398	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
399	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
400	static FNVMXEXITHANDLER hmR0VmxExitMwait;
401	static FNVMXEXITHANDLER hmR0VmxExitMtf;
402	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
403	static FNVMXEXITHANDLER hmR0VmxExitPause;
404	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
405	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
406	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
407	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
408	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
409	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
410	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
411	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
412	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
413	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
414	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
415	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
416	/** @} */
417
418	/** @name Helpers for hardware exceptions VM-exit handlers.
419	* @{
420	*/
421	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
422	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
423	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
424	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
425	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
426	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
427	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
428	/** @} */
429
430
431	/*********************************************************************************************************************************
432	* Global Variables *
433	*********************************************************************************************************************************/
434	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
435	static const uint32_t g_aVmcsCacheSegBase[] =
436	{
437	VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
438	VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
439	VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
440	VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
441	VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
442	VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
443	};
444	AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
445	#endif
446	static const uint32_t g_aVmcsSegBase[] =
447	{
448	VMX_VMCS_GUEST_ES_BASE,
449	VMX_VMCS_GUEST_CS_BASE,
450	VMX_VMCS_GUEST_SS_BASE,
451	VMX_VMCS_GUEST_DS_BASE,
452	VMX_VMCS_GUEST_FS_BASE,
453	VMX_VMCS_GUEST_GS_BASE
454	};
455	static const uint32_t g_aVmcsSegSel[] =
456	{
457	VMX_VMCS16_GUEST_ES_SEL,
458	VMX_VMCS16_GUEST_CS_SEL,
459	VMX_VMCS16_GUEST_SS_SEL,
460	VMX_VMCS16_GUEST_DS_SEL,
461	VMX_VMCS16_GUEST_FS_SEL,
462	VMX_VMCS16_GUEST_GS_SEL
463	};
464	static const uint32_t g_aVmcsSegLimit[] =
465	{
466	VMX_VMCS32_GUEST_ES_LIMIT,
467	VMX_VMCS32_GUEST_CS_LIMIT,
468	VMX_VMCS32_GUEST_SS_LIMIT,
469	VMX_VMCS32_GUEST_DS_LIMIT,
470	VMX_VMCS32_GUEST_FS_LIMIT,
471	VMX_VMCS32_GUEST_GS_LIMIT
472	};
473	static const uint32_t g_aVmcsSegAttr[] =
474	{
475	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
476	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
477	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
478	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
479	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
480	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
481	};
482	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
483	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
484	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
485	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
486
487	#ifdef HMVMX_USE_FUNCTION_TABLE
488	/**
489	* VMX_EXIT dispatch table.
490	*/
491	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
492	{
493	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
494	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
495	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
496	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
497	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
498	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
499	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
500	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
501	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
502	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
503	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
504	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
505	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
506	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
507	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
508	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
509	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
510	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
511	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
512	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
513	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
514	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
515	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
516	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
517	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
518	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
519	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
520	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
521	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
522	#else
523	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
524	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
525	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
526	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
527	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
528	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
529	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
530	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
531	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
532	#endif
533	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
534	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
535	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
536	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
537	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
538	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
539	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
540	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
541	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
542	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
543	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
544	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
545	/* 40 UNDEFINED */ hmR0VmxExitPause,
546	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
547	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
548	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
549	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
550	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
551	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitXdtrAccess,
552	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitXdtrAccess,
553	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
554	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
555	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
556	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
557	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
558	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
559	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
560	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
561	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
562	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
563	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
564	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
565	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
566	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
567	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
568	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
569	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
570	};
571	#endif /* HMVMX_USE_FUNCTION_TABLE */
572
573	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
574	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
575	{
576	/* 0 */ "(Not Used)",
577	/* 1 */ "VMCALL executed in VMX root operation.",
578	/* 2 */ "VMCLEAR with invalid physical address.",
579	/* 3 */ "VMCLEAR with VMXON pointer.",
580	/* 4 */ "VMLAUNCH with non-clear VMCS.",
581	/* 5 */ "VMRESUME with non-launched VMCS.",
582	/* 6 */ "VMRESUME after VMXOFF",
583	/* 7 */ "VM-entry with invalid control fields.",
584	/* 8 */ "VM-entry with invalid host state fields.",
585	/* 9 */ "VMPTRLD with invalid physical address.",
586	/* 10 */ "VMPTRLD with VMXON pointer.",
587	/* 11 */ "VMPTRLD with incorrect revision identifier.",
588	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
589	/* 13 */ "VMWRITE to read-only VMCS component.",
590	/* 14 */ "(Not Used)",
591	/* 15 */ "VMXON executed in VMX root operation.",
592	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
593	/* 17 */ "VM-entry with non-launched executing VMCS.",
594	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
595	/* 19 */ "VMCALL with non-clear VMCS.",
596	/* 20 */ "VMCALL with invalid VM-exit control fields.",
597	/* 21 */ "(Not Used)",
598	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
599	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
600	/* 24 */ "VMCALL with invalid SMM-monitor features.",
601	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
602	/* 26 */ "VM-entry with events blocked by MOV SS.",
603	/* 27 */ "(Not Used)",
604	/* 28 */ "Invalid operand to INVEPT/INVVPID."
605	};
606	#endif /* VBOX_STRICT */
607
608
609	/**
610	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
611	*
612	* Any bit set in this mask is owned by the host/hypervisor and would cause a
613	* VM-exit when modified by the guest.
614	*
615	* @returns The static CR0 guest/host mask.
616	* @param pVCpu The cross context virtual CPU structure.
617	*/
618	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PCVMCPU pVCpu)
619	{
620	/*
621	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
622	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
623	*/
624	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
625	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
626	* and @bugref{6944}. */
627	PVM pVM = pVCpu->CTX_SUFF(pVM);
628	return ( X86_CR0_PE
629	\| X86_CR0_NE
630	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
631	\| X86_CR0_PG
632	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
633	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
634	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
635	}
636
637
638	/**
639	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
640	*
641	* Any bit set in this mask is owned by the host/hypervisor and would cause a
642	* VM-exit when modified by the guest.
643	*
644	* @returns The static CR4 guest/host mask.
645	* @param pVCpu The cross context virtual CPU structure.
646	*/
647	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PCVMCPU pVCpu)
648	{
649	/*
650	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
651	* these bits are reserved on hardware that does not support them. Since the
652	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
653	* these bits and handle it depending on whether we expose them to the guest.
654	*/
655	PVM pVM = pVCpu->CTX_SUFF(pVM);
656	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
657	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
658	return ( X86_CR4_VMXE
659	\| X86_CR4_VME
660	\| X86_CR4_PAE
661	\| X86_CR4_PGE
662	\| X86_CR4_PSE
663	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
664	\| (fPcid ? X86_CR4_PCIDE : 0));
665	}
666
667
668	/**
669	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
670	* area.
671	*
672	* @returns @c true if it's different, @c false otherwise.
673	* @param pVmcsInfo The VMCS info. object.
674	*/
675	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
676	{
677	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
678	&& pVmcsInfo->pvGuestMsrStore);
679	}
680
681
682	/**
683	* Adds one or more exceptions to the exception bitmap and commits it to the current
684	* VMCS.
685	*
686	* @returns VBox status code.
687	* @param pVmxTransient The VMX-transient structure.
688	* @param uXcptMask The exception(s) to add.
689	*/
690	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
691	{
692	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
693	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
694	if ((uXcptBitmap & uXcptMask) != uXcptMask)
695	{
696	uXcptBitmap \|= uXcptMask;
697	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
698	AssertRCReturn(rc, rc);
699	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
700	}
701	return VINF_SUCCESS;
702	}
703
704
705	/**
706	* Adds an exception to the exception bitmap and commits it to the current VMCS.
707	*
708	* @returns VBox status code.
709	* @param pVmxTransient The VMX-transient structure.
710	* @param uXcpt The exception to add.
711	*/
712	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
713	{
714	Assert(uXcpt <= X86_XCPT_LAST);
715	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
716	}
717
718
719	/**
720	* Remove one or more exceptions from the exception bitmap and commits it to the
721	* current VMCS.
722	*
723	* This takes care of not removing the exception intercept if a nested-guest
724	* requires the exception to be intercepted.
725	*
726	* @returns VBox status code.
727	* @param pVCpu The cross context virtual CPU structure.
728	* @param pVmxTransient The VMX-transient structure.
729	* @param uXcptMask The exception(s) to remove.
730	*/
731	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
732	{
733	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
734	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
735	if (u32XcptBitmap & uXcptMask)
736	{
737	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
738	if (!pVmxTransient->fIsNestedGuest)
739	{ /* likely */ }
740	else
741	{
742	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
743	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
744	}
745	#endif
746	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
747	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
748	\| RT_BIT(X86_XCPT_DE)
749	\| RT_BIT(X86_XCPT_NM)
750	\| RT_BIT(X86_XCPT_TS)
751	\| RT_BIT(X86_XCPT_UD)
752	\| RT_BIT(X86_XCPT_NP)
753	\| RT_BIT(X86_XCPT_SS)
754	\| RT_BIT(X86_XCPT_GP)
755	\| RT_BIT(X86_XCPT_PF)
756	\| RT_BIT(X86_XCPT_MF));
757	#elif defined(HMVMX_ALWAYS_TRAP_PF)
758	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
759	#endif
760	if (uXcptMask)
761	{
762	/* Validate we are not removing any essential exception intercepts. */
763	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
764	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
765	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
766
767	/* Remove it from the exception bitmap. */
768	u32XcptBitmap &= ~uXcptMask;
769
770	/* Commit and update the cache if necessary. */
771	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
772	{
773	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
774	AssertRCReturn(rc, rc);
775	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
776	}
777	}
778	}
779	return VINF_SUCCESS;
780	}
781
782
783	/**
784	* Remove an exceptions from the exception bitmap and commits it to the current
785	* VMCS.
786	*
787	* @returns VBox status code.
788	* @param pVCpu The cross context virtual CPU structure.
789	* @param pVmxTransient The VMX-transient structure.
790	* @param uXcpt The exception to remove.
791	*/
792	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
793	{
794	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
795	}
796
797
798	/**
799	* Loads the VMCS specified by the VMCS info. object.
800	*
801	* @returns VBox status code.
802	* @param pVmcsInfo The VMCS info. object.
803	*/
804	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
805	{
806	Assert(pVmcsInfo);
807	Assert(pVmcsInfo->HCPhysVmcs);
808	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
809
810	if (pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_CLEAR)
811	{
812	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
813	if (RT_SUCCESS(rc))
814	{
815	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
816	return VINF_SUCCESS;
817	}
818	return rc;
819	}
820	return VERR_VMX_INVALID_VMCS_LAUNCH_STATE;
821	}
822
823
824	/**
825	* Clears the VMCS specified by the VMCS info. object.
826	*
827	* @returns VBox status code.
828	* @param pVmcsInfo The VMCS info. object.
829	*/
830	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
831	{
832	Assert(pVmcsInfo);
833	Assert(pVmcsInfo->HCPhysVmcs);
834	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
835
836	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
837	if (RT_SUCCESS(rc))
838	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
839	return rc;
840	}
841
842
843	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
844	/**
845	* Switches the current VMCS to the one specified.
846	*
847	* @returns VBox status code.
848	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
849	* @param pVmcsInfoTo The VMCS info. object we are switching to.
850	*
851	* @remarks Called with interrupts disabled.
852	*/
853	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
854	{
855	Assert(pVmcsInfoFrom);
856	Assert(pVmcsInfoTo);
857
858	/*
859	* Clear the VMCS we are switching out if it has not already been cleared.
860	* This will sync any CPU internal data back to the VMCS.
861	*/
862	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
863	{
864	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
865	if (RT_SUCCESS(rc))
866	{ /* likely */ }
867	else
868	return rc;
869	}
870
871	/*
872	* Clear the VMCS we are switching to if it has not already been cleared.
873	* This will initialize the VMCS launch state to "clear" required for loading it.
874	*
875	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
876	*/
877	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
878	{
879	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
880	if (RT_SUCCESS(rc))
881	{ /* likely */ }
882	else
883	return rc;
884	}
885
886	/*
887	* Finally, load the VMCS we are switching to.
888	*/
889	return hmR0VmxLoadVmcs(pVmcsInfoTo);
890	}
891	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
892
893
894	/**
895	* Updates the VM's last error record.
896	*
897	* If there was a VMX instruction error, reads the error data from the VMCS and
898	* updates VCPU's last error record as well.
899	*
900	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
901	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
902	* VERR_VMX_INVALID_VMCS_FIELD.
903	* @param rc The error code.
904	*/
905	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
906	{
907	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
908	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
909	{
910	AssertPtrReturnVoid(pVCpu);
911	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
912	}
913	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
914	}
915
916
917	#ifdef VBOX_STRICT
918	/**
919	* Reads the VM-entry interruption-information field from the VMCS into the VMX
920	* transient structure.
921	*
922	* @returns VBox status code.
923	* @param pVmxTransient The VMX-transient structure.
924	*
925	* @remarks No-long-jump zone!!!
926	*/
927	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
928	{
929	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
930	AssertRCReturn(rc, rc);
931	return VINF_SUCCESS;
932	}
933
934
935	/**
936	* Reads the VM-entry exception error code field from the VMCS into
937	* the VMX transient structure.
938	*
939	* @returns VBox status code.
940	* @param pVmxTransient The VMX-transient structure.
941	*
942	* @remarks No-long-jump zone!!!
943	*/
944	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
945	{
946	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
947	AssertRCReturn(rc, rc);
948	return VINF_SUCCESS;
949	}
950
951
952	/**
953	* Reads the VM-entry exception error code field from the VMCS into
954	* the VMX transient structure.
955	*
956	* @returns VBox status code.
957	* @param pVmxTransient The VMX-transient structure.
958	*
959	* @remarks No-long-jump zone!!!
960	*/
961	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
962	{
963	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
964	AssertRCReturn(rc, rc);
965	return VINF_SUCCESS;
966	}
967	#endif /* VBOX_STRICT */
968
969
970	/**
971	* Reads the VM-exit interruption-information field from the VMCS into the VMX
972	* transient structure.
973	*
974	* @returns VBox status code.
975	* @param pVmxTransient The VMX-transient structure.
976	*/
977	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
978	{
979	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
980	{
981	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
982	AssertRCReturn(rc,rc);
983	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
984	}
985	return VINF_SUCCESS;
986	}
987
988
989	/**
990	* Reads the VM-exit interruption error code from the VMCS into the VMX
991	* transient structure.
992	*
993	* @returns VBox status code.
994	* @param pVmxTransient The VMX-transient structure.
995	*/
996	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
997	{
998	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
999	{
1000	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1001	AssertRCReturn(rc, rc);
1002	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1003	}
1004	return VINF_SUCCESS;
1005	}
1006
1007
1008	/**
1009	* Reads the VM-exit instruction length field from the VMCS into the VMX
1010	* transient structure.
1011	*
1012	* @returns VBox status code.
1013	* @param pVmxTransient The VMX-transient structure.
1014	*/
1015	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1016	{
1017	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1018	{
1019	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1020	AssertRCReturn(rc, rc);
1021	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1022	}
1023	return VINF_SUCCESS;
1024	}
1025
1026
1027	/**
1028	* Reads the VM-exit instruction-information field from the VMCS into
1029	* the VMX transient structure.
1030	*
1031	* @returns VBox status code.
1032	* @param pVmxTransient The VMX-transient structure.
1033	*/
1034	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1035	{
1036	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1037	{
1038	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1039	AssertRCReturn(rc, rc);
1040	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1041	}
1042	return VINF_SUCCESS;
1043	}
1044
1045
1046	/**
1047	* Reads the VM-exit Qualification from the VMCS into the VMX transient structure.
1048	*
1049	* @returns VBox status code.
1050	* @param pVCpu The cross context virtual CPU structure of the
1051	* calling EMT. (Required for the VMCS cache case.)
1052	* @param pVmxTransient The VMX-transient structure.
1053	*/
1054	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1055	{
1056	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1057	{
1058	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1059	AssertRCReturn(rc, rc);
1060	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1061	}
1062	return VINF_SUCCESS;
1063	}
1064
1065
1066	/**
1067	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1068	*
1069	* @returns VBox status code.
1070	* @param pVCpu The cross context virtual CPU structure of the
1071	* calling EMT. (Required for the VMCS cache case.)
1072	* @param pVmxTransient The VMX-transient structure.
1073	*/
1074	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1075	{
1076	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1077	{
1078	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1079	AssertRCReturn(rc, rc);
1080	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1081	}
1082	return VINF_SUCCESS;
1083	}
1084
1085
1086	/**
1087	* Reads the IDT-vectoring information field from the VMCS into the VMX
1088	* transient structure.
1089	*
1090	* @returns VBox status code.
1091	* @param pVmxTransient The VMX-transient structure.
1092	*
1093	* @remarks No-long-jump zone!!!
1094	*/
1095	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1096	{
1097	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1098	{
1099	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1100	AssertRCReturn(rc, rc);
1101	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1102	}
1103	return VINF_SUCCESS;
1104	}
1105
1106
1107	/**
1108	* Reads the IDT-vectoring error code from the VMCS into the VMX
1109	* transient structure.
1110	*
1111	* @returns VBox status code.
1112	* @param pVmxTransient The VMX-transient structure.
1113	*/
1114	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1115	{
1116	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1117	{
1118	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1119	AssertRCReturn(rc, rc);
1120	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1121	}
1122	return VINF_SUCCESS;
1123	}
1124
1125
1126	/**
1127	* Enters VMX root mode operation on the current CPU.
1128	*
1129	* @returns VBox status code.
1130	* @param pVM The cross context VM structure. Can be
1131	* NULL, after a resume.
1132	* @param HCPhysCpuPage Physical address of the VMXON region.
1133	* @param pvCpuPage Pointer to the VMXON region.
1134	*/
1135	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1136	{
1137	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1138	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1139	Assert(pvCpuPage);
1140	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1141
1142	if (pVM)
1143	{
1144	/* Write the VMCS revision identifier to the VMXON region. */
1145	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1146	}
1147
1148	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1149	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1150
1151	/* Enable the VMX bit in CR4 if necessary. */
1152	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1153
1154	/* Enter VMX root mode. */
1155	int rc = VMXEnable(HCPhysCpuPage);
1156	if (RT_FAILURE(rc))
1157	{
1158	if (!(uOldCr4 & X86_CR4_VMXE))
1159	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1160
1161	if (pVM)
1162	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1163	}
1164
1165	/* Restore interrupts. */
1166	ASMSetFlags(fEFlags);
1167	return rc;
1168	}
1169
1170
1171	/**
1172	* Exits VMX root mode operation on the current CPU.
1173	*
1174	* @returns VBox status code.
1175	*/
1176	static int hmR0VmxLeaveRootMode(void)
1177	{
1178	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1179
1180	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1181	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1182
1183	/* If we're for some reason not in VMX root mode, then don't leave it. */
1184	RTCCUINTREG const uHostCR4 = ASMGetCR4();
1185
1186	int rc;
1187	if (uHostCR4 & X86_CR4_VMXE)
1188	{
1189	/* Exit VMX root mode and clear the VMX bit in CR4. */
1190	VMXDisable();
1191	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1192	rc = VINF_SUCCESS;
1193	}
1194	else
1195	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1196
1197	/* Restore interrupts. */
1198	ASMSetFlags(fEFlags);
1199	return rc;
1200	}
1201
1202
1203	/**
1204	* Allocates and maps a physically contiguous page. The allocated page is
1205	* zero'd out (used by various VT-x structures).
1206	*
1207	* @returns IPRT status code.
1208	* @param pMemObj Pointer to the ring-0 memory object.
1209	* @param ppVirt Where to store the virtual address of the
1210	* allocation.
1211	* @param pHCPhys Where to store the physical address of the
1212	* allocation.
1213	*/
1214	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1215	{
1216	AssertPtr(pMemObj);
1217	AssertPtr(ppVirt);
1218	AssertPtr(pHCPhys);
1219	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1220	if (RT_FAILURE(rc))
1221	return rc;
1222	ppVirt = RTR0MemObjAddress(pMemObj);
1223	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1224	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1225	return VINF_SUCCESS;
1226	}
1227
1228
1229	/**
1230	* Frees and unmaps an allocated, physical page.
1231	*
1232	* @param pMemObj Pointer to the ring-0 memory object.
1233	* @param ppVirt Where to re-initialize the virtual address of
1234	* allocation as 0.
1235	* @param pHCPhys Where to re-initialize the physical address of the
1236	* allocation as 0.
1237	*/
1238	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1239	{
1240	AssertPtr(pMemObj);
1241	AssertPtr(ppVirt);
1242	AssertPtr(pHCPhys);
1243	/* NULL is valid, accepted and ignored by the free function below. */
1244	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1245	*pMemObj = NIL_RTR0MEMOBJ;
1246	*ppVirt = NULL;
1247	*pHCPhys = NIL_RTHCPHYS;
1248	}
1249
1250
1251	/**
1252	* Initializes a VMCS info. object.
1253	*
1254	* @param pVmcsInfo The VMCS info. object.
1255	*/
1256	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1257	{
1258	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1259
1260	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1261	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1262	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1263	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1264	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1265	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1266	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1267	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1268	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1269	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1270	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1271	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1272	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1273	}
1274
1275
1276	/**
1277	* Frees the VT-x structures for a VMCS info. object.
1278	*
1279	* @param pVM The cross context VM structure.
1280	* @param pVmcsInfo The VMCS info. object.
1281	*/
1282	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1283	{
1284	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1285
1286	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1287	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1288
1289	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1290	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1291	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1292
1293	hmR0VmxInitVmcsInfo(pVmcsInfo);
1294	}
1295
1296
1297	/**
1298	* Allocates the VT-x structures for a VMCS info. object.
1299	*
1300	* @returns VBox status code.
1301	* @param pVCpu The cross context virtual CPU structure.
1302	* @param pVmcsInfo The VMCS info. object.
1303	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1304	*/
1305	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1306	{
1307	PVM pVM = pVCpu->CTX_SUFF(pVM);
1308
1309	/* Allocate the guest VM control structure (VMCS). */
1310	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1311	if (RT_SUCCESS(rc))
1312	{
1313	if (!fIsNstGstVmcs)
1314	{
1315	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1316	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1317	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1318	{
1319	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1320	NULL /* pR3Ptr /, NULL / pRCPtr */);
1321	}
1322	}
1323	else
1324	{
1325	Assert(pVmcsInfo->HCPhysVirtApic == NIL_RTHCPHYS);
1326	Assert(!pVmcsInfo->pbVirtApic);
1327	}
1328
1329	if (RT_SUCCESS(rc))
1330	{
1331	/*
1332	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1333	* transparent accesses of specific MSRs.
1334	*
1335	* If the condition for enabling MSR bitmaps changes here, don't forget to
1336	* update HMIsMsrBitmapActive().
1337	*
1338	* We don't share MSR bitmaps between the guest and nested-guest as we then
1339	* don't need to care about carefully restoring the guest MSR bitmap.
1340	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1341	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1342	*/
1343	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1344	{
1345	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1346	if (RT_SUCCESS(rc))
1347	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1348	}
1349
1350	if (RT_SUCCESS(rc))
1351	{
1352	/*
1353	* Allocate the VM-entry MSR-load area for the guest MSRs.
1354	*
1355	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1356	* the guest and nested-guest.
1357	*/
1358	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1359	&pVmcsInfo->HCPhysGuestMsrLoad);
1360	if (RT_SUCCESS(rc))
1361	{
1362	/*
1363	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1364	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1365	*/
1366	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1367	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1368	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1369
1370	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1371	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1372	&pVmcsInfo->HCPhysHostMsrLoad);
1373	}
1374	}
1375	}
1376	}
1377
1378	return rc;
1379	}
1380
1381
1382	/**
1383	* Free all VT-x structures for the VM.
1384	*
1385	* @returns IPRT status code.
1386	* @param pVM The cross context VM structure.
1387	*/
1388	static void hmR0VmxStructsFree(PVM pVM)
1389	{
1390	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1391	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1392	#endif
1393	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1394
1395	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1396	{
1397	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1398	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1399	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1400	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1401	if (pVM->cpum.ro.GuestFeatures.fVmx)
1402	{
1403	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1404	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1405	}
1406	#endif
1407	}
1408	}
1409
1410
1411	/**
1412	* Allocate all VT-x structures for the VM.
1413	*
1414	* @returns IPRT status code.
1415	* @param pVM The cross context VM structure.
1416	*/
1417	static int hmR0VmxStructsAlloc(PVM pVM)
1418	{
1419	/*
1420	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1421	* The VMCS size cannot be more than 4096 bytes.
1422	*
1423	* See Intel spec. Appendix A.1 "Basic VMX Information".
1424	*/
1425	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1426	if (cbVmcs <= X86_PAGE_4K_SIZE)
1427	{ /* likely */ }
1428	else
1429	{
1430	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1431	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1432	}
1433
1434	/*
1435	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1436	*/
1437	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1438	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1439	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1440	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1441	#endif
1442
1443	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1444	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1445	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1446
1447	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1448	{
1449	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1450	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1451	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1452	}
1453
1454	/*
1455	* Allocate per-VM VT-x structures.
1456	*/
1457	int rc = VINF_SUCCESS;
1458	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1459	/* Allocate crash-dump magic scratch page. */
1460	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1461	if (RT_FAILURE(rc))
1462	{
1463	hmR0VmxStructsFree(pVM);
1464	return rc;
1465	}
1466	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1467	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1468	#endif
1469
1470	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1471	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1472	{
1473	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1474	&pVM->hm.s.vmx.HCPhysApicAccess);
1475	if (RT_FAILURE(rc))
1476	{
1477	hmR0VmxStructsFree(pVM);
1478	return rc;
1479	}
1480	}
1481
1482	/*
1483	* Initialize per-VCPU VT-x structures.
1484	*/
1485	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1486	{
1487	/* Allocate the guest VMCS structures. */
1488	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1489	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
1490	if (RT_SUCCESS(rc))
1491	{
1492	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1493	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
1494	if (pVM->cpum.ro.GuestFeatures.fVmx)
1495	{
1496	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
1497	if (RT_SUCCESS(rc))
1498	{ /* likely */ }
1499	else
1500	break;
1501	}
1502	#endif
1503	}
1504	else
1505	break;
1506	}
1507
1508	if (RT_FAILURE(rc))
1509	{
1510	hmR0VmxStructsFree(pVM);
1511	return rc;
1512	}
1513
1514	return VINF_SUCCESS;
1515	}
1516
1517
1518	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1519	/**
1520	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
1521	*
1522	* @returns @c true if the MSR is intercepted, @c false otherwise.
1523	* @param pvMsrBitmap The MSR bitmap.
1524	* @param offMsr The MSR byte offset.
1525	* @param iBit The bit offset from the byte offset.
1526	*/
1527	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
1528	{
1529	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
1530	Assert(pbMsrBitmap);
1531	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
1532	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
1533	}
1534	#endif
1535
1536
1537	/**
1538	* Sets the permission bits for the specified MSR in the given MSR bitmap.
1539	*
1540	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
1541	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
1542	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
1543	* the read/write access of this MSR.
1544	*
1545	* @param pVCpu The cross context virtual CPU structure.
1546	* @param pVmcsInfo The VMCS info. object.
1547	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1548	* @param idMsr The MSR value.
1549	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
1550	* include both a read -and- a write permission!
1551	*
1552	* @sa CPUMGetVmxMsrPermission.
1553	* @remarks Can be called with interrupts disabled.
1554	*/
1555	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
1556	{
1557	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
1558	Assert(pbMsrBitmap);
1559	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
1560
1561	/*
1562	* MSR-bitmap Layout:
1563	* Byte index MSR range Interpreted as
1564	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1565	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1566	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1567	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1568	*
1569	* A bit corresponding to an MSR within the above range causes a VM-exit
1570	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
1571	* the MSR range, it always cause a VM-exit.
1572	*
1573	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
1574	*/
1575	uint16_t const offBitmapRead = 0;
1576	uint16_t const offBitmapWrite = 0x800;
1577	uint16_t offMsr;
1578	int32_t iBit;
1579	if (idMsr <= UINT32_C(0x00001fff))
1580	{
1581	offMsr = 0;
1582	iBit = idMsr;
1583	}
1584	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1585	{
1586	offMsr = 0x400;
1587	iBit = idMsr - UINT32_C(0xc0000000);
1588	}
1589	else
1590	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
1591
1592	/*
1593	* Set the MSR read permission.
1594	*/
1595	uint16_t const offMsrRead = offBitmapRead + offMsr;
1596	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
1597	if (fMsrpm & VMXMSRPM_ALLOW_RD)
1598	{
1599	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1600	bool const fClear = !fIsNstGstVmcs ? true
1601	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
1602	#else
1603	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1604	bool const fClear = true;
1605	#endif
1606	if (fClear)
1607	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
1608	}
1609	else
1610	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
1611
1612	/*
1613	* Set the MSR write permission.
1614	*/
1615	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
1616	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
1617	if (fMsrpm & VMXMSRPM_ALLOW_WR)
1618	{
1619	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1620	bool const fClear = !fIsNstGstVmcs ? true
1621	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
1622	#else
1623	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1624	bool const fClear = true;
1625	#endif
1626	if (fClear)
1627	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
1628	}
1629	else
1630	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
1631	}
1632
1633
1634	/**
1635	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1636	* area.
1637	*
1638	* @returns VBox status code.
1639	* @param pVCpu The cross context virtual CPU structure.
1640	* @param pVmcsInfo The VMCS info. object.
1641	* @param cMsrs The number of MSRs.
1642	*/
1643	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
1644	{
1645	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1646	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1647	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
1648	{
1649	/* Commit the MSR counts to the VMCS and update the cache. */
1650	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
1651	{
1652	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1653	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1654	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1655	AssertRCReturn(rc, rc);
1656
1657	pVmcsInfo->cEntryMsrLoad = cMsrs;
1658	pVmcsInfo->cExitMsrStore = cMsrs;
1659	pVmcsInfo->cExitMsrLoad = cMsrs;
1660	}
1661	return VINF_SUCCESS;
1662	}
1663
1664	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
1665	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1666	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1667	}
1668
1669
1670	/**
1671	* Adds a new (or updates the value of an existing) guest/host MSR
1672	* pair to be swapped during the world-switch as part of the
1673	* auto-load/store MSR area in the VMCS.
1674	*
1675	* @returns VBox status code.
1676	* @param pVCpu The cross context virtual CPU structure.
1677	* @param pVmxTransient The VMX-transient structure.
1678	* @param idMsr The MSR.
1679	* @param uGuestMsrValue Value of the guest MSR.
1680	* @param fSetReadWrite Whether to set the guest read/write access of this
1681	* MSR (thus not causing a VM-exit).
1682	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1683	* necessary.
1684	*/
1685	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
1686	bool fSetReadWrite, bool fUpdateHostMsr)
1687	{
1688	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1689	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1690	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1691	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1692	uint32_t i;
1693
1694	/* Paranoia. */
1695	Assert(pGuestMsrLoad);
1696
1697	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
1698
1699	/* Check if the MSR already exists in the VM-entry MSR-load area. */
1700	for (i = 0; i < cMsrs; i++)
1701	{
1702	if (pGuestMsrLoad[i].u32Msr == idMsr)
1703	break;
1704	}
1705
1706	bool fAdded = false;
1707	if (i == cMsrs)
1708	{
1709	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
1710	++cMsrs;
1711	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1712	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
1713
1714	/* Set the guest to read/write this MSR without causing VM-exits. */
1715	if ( fSetReadWrite
1716	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
1717	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
1718
1719	LogFlowFunc(("MSR added, cMsrs now %u\n", cMsrs));
1720	fAdded = true;
1721	}
1722
1723	/* Update the MSR value for the newly added or already existing MSR. */
1724	pGuestMsrLoad[i].u32Msr = idMsr;
1725	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
1726
1727	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
1728	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1729	{
1730	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1731	pGuestMsrStore[i].u32Msr = idMsr;
1732	pGuestMsrStore[i].u64Value = uGuestMsrValue;
1733	}
1734
1735	/* Update the corresponding slot in the host MSR area. */
1736	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1737	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
1738	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
1739	pHostMsr[i].u32Msr = idMsr;
1740
1741	/*
1742	* Only if the caller requests to update the host MSR value AND we've newly added the
1743	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
1744	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
1745	*
1746	* We do this for performance reasons since reading MSRs may be quite expensive.
1747	*/
1748	if (fAdded)
1749	{
1750	if (fUpdateHostMsr)
1751	{
1752	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1753	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1754	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
1755	}
1756	else
1757	{
1758	/* Someone else can do the work. */
1759	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
1760	}
1761	}
1762	return VINF_SUCCESS;
1763	}
1764
1765
1766	/**
1767	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1768	* auto-load/store MSR area in the VMCS.
1769	*
1770	* @returns VBox status code.
1771	* @param pVCpu The cross context virtual CPU structure.
1772	* @param pVmxTransient The VMX-transient structure.
1773	* @param idMsr The MSR.
1774	*/
1775	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
1776	{
1777	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1778	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1779	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1780	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1781
1782	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
1783
1784	for (uint32_t i = 0; i < cMsrs; i++)
1785	{
1786	/* Find the MSR. */
1787	if (pGuestMsrLoad[i].u32Msr == idMsr)
1788	{
1789	/*
1790	* If it's the last MSR, we only need to reduce the MSR count.
1791	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
1792	*/
1793	if (i < cMsrs - 1)
1794	{
1795	/* Remove it from the VM-entry MSR-load area. */
1796	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
1797	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
1798
1799	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
1800	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1801	{
1802	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1803	Assert(pGuestMsrStore[i].u32Msr == idMsr);
1804	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
1805	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
1806	}
1807
1808	/* Remove it from the VM-exit MSR-load area. */
1809	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1810	Assert(pHostMsr[i].u32Msr == idMsr);
1811	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
1812	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
1813	}
1814
1815	/* Reduce the count to reflect the removed MSR and bail. */
1816	--cMsrs;
1817	break;
1818	}
1819	}
1820
1821	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
1822	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
1823	{
1824	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1825	AssertRCReturn(rc, rc);
1826
1827	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1828	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1829	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
1830
1831	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
1832	return VINF_SUCCESS;
1833	}
1834
1835	return VERR_NOT_FOUND;
1836	}
1837
1838
1839	/**
1840	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
1841	*
1842	* @returns @c true if found, @c false otherwise.
1843	* @param pVmcsInfo The VMCS info. object.
1844	* @param idMsr The MSR to find.
1845	*/
1846	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
1847	{
1848	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1849	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
1850	Assert(pMsrs);
1851	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
1852	for (uint32_t i = 0; i < cMsrs; i++)
1853	{
1854	if (pMsrs[i].u32Msr == idMsr)
1855	return true;
1856	}
1857	return false;
1858	}
1859
1860
1861	/**
1862	* Updates the value of all host MSRs in the VM-exit MSR-load area.
1863	*
1864	* @param pVCpu The cross context virtual CPU structure.
1865	* @param pVmcsInfo The VMCS info. object.
1866	*
1867	* @remarks No-long-jump zone!!!
1868	*/
1869	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
1870	{
1871	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1872
1873	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1874	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
1875	Assert(pHostMsrLoad);
1876	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
1877	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
1878	for (uint32_t i = 0; i < cMsrs; i++)
1879	{
1880	/*
1881	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1882	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1883	*/
1884	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
1885	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
1886	else
1887	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
1888	}
1889	}
1890
1891
1892	/**
1893	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1894	* perform lazy restoration of the host MSRs while leaving VT-x.
1895	*
1896	* @param pVCpu The cross context virtual CPU structure.
1897	*
1898	* @remarks No-long-jump zone!!!
1899	*/
1900	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1901	{
1902	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1903
1904	/*
1905	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
1906	*/
1907	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1908	{
1909	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1910	#if HC_ARCH_BITS == 64
1911	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1912	{
1913	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
1914	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
1915	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
1916	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1917	}
1918	#endif
1919	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1920	}
1921	}
1922
1923
1924	/**
1925	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1926	* lazily while leaving VT-x.
1927	*
1928	* @returns true if it does, false otherwise.
1929	* @param pVCpu The cross context virtual CPU structure.
1930	* @param idMsr The MSR to check.
1931	*/
1932	static bool hmR0VmxIsLazyGuestMsr(PCVMCPU pVCpu, uint32_t idMsr)
1933	{
1934	NOREF(pVCpu);
1935	#if HC_ARCH_BITS == 64
1936	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1937	{
1938	switch (idMsr)
1939	{
1940	case MSR_K8_LSTAR:
1941	case MSR_K6_STAR:
1942	case MSR_K8_SF_MASK:
1943	case MSR_K8_KERNEL_GS_BASE:
1944	return true;
1945	}
1946	}
1947	#else
1948	RT_NOREF(pVCpu, idMsr);
1949	#endif
1950	return false;
1951	}
1952
1953
1954	/**
1955	* Loads a set of guests MSRs to allow read/passthru to the guest.
1956	*
1957	* The name of this function is slightly confusing. This function does NOT
1958	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1959	* common prefix for functions dealing with "lazy restoration" of the shared
1960	* MSRs.
1961	*
1962	* @param pVCpu The cross context virtual CPU structure.
1963	*
1964	* @remarks No-long-jump zone!!!
1965	*/
1966	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1967	{
1968	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1969	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1970
1971	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1972	#if HC_ARCH_BITS == 64
1973	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1974	{
1975	/*
1976	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1977	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1978	* we can skip a few MSR writes.
1979	*
1980	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1981	* guest MSR values in the guest-CPU context might be different to what's currently
1982	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1983	* CPU, see @bugref{8728}.
1984	*/
1985	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1986	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1987	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
1988	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
1989	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
1990	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
1991	{
1992	#ifdef VBOX_STRICT
1993	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1994	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1995	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1996	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1997	#endif
1998	}
1999	else
2000	{
2001	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2002	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2003	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2004	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2005	}
2006	}
2007	#endif
2008	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2009	}
2010
2011
2012	/**
2013	* Performs lazy restoration of the set of host MSRs if they were previously
2014	* loaded with guest MSR values.
2015	*
2016	* @param pVCpu The cross context virtual CPU structure.
2017	*
2018	* @remarks No-long-jump zone!!!
2019	* @remarks The guest MSRs should have been saved back into the guest-CPU
2020	* context by hmR0VmxImportGuestState()!!!
2021	*/
2022	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2023	{
2024	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2025	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2026
2027	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2028	{
2029	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2030	#if HC_ARCH_BITS == 64
2031	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2032	{
2033	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2034	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2035	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2036	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2037	}
2038	#endif
2039	}
2040	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2041	}
2042
2043
2044	/**
2045	* Verifies that our cached values of the VMCS fields are all consistent with
2046	* what's actually present in the VMCS.
2047	*
2048	* @returns VBox status code.
2049	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2050	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2051	* VMCS content. HMCPU error-field is
2052	* updated, see VMX_VCI_XXX.
2053	* @param pVCpu The cross context virtual CPU structure.
2054	* @param pVmcsInfo The VMCS info. object.
2055	*/
2056	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2057	{
2058	uint32_t u32Val;
2059	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2060	AssertRCReturn(rc, rc);
2061	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2062	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32EntryCtls, u32Val),
2063	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2064	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2065
2066	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2067	AssertRCReturn(rc, rc);
2068	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2069	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ExitCtls, u32Val),
2070	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2071	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2072
2073	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2074	AssertRCReturn(rc, rc);
2075	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2076	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32PinCtls, u32Val),
2077	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2078	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2079
2080	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2081	AssertRCReturn(rc, rc);
2082	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2083	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls, u32Val),
2084	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2085	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2086
2087	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2088	{
2089	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2090	AssertRCReturn(rc, rc);
2091	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2092	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls2, u32Val),
2093	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2094	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2095	}
2096
2097	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2098	AssertRCReturn(rc, rc);
2099	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2100	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32XcptBitmap, u32Val),
2101	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2102	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2103
2104	uint64_t u64Val;
2105	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2106	AssertRCReturn(rc, rc);
2107	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2108	("Cache=%#RX64 VMCS=%#RX64\n", pVmcsInfo->u64TscOffset, u64Val),
2109	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2110	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2111
2112	return VINF_SUCCESS;
2113	}
2114
2115
2116	#ifdef VBOX_STRICT
2117	/**
2118	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2119	*
2120	* @param pVCpu The cross context virtual CPU structure.
2121	* @param pVmcsInfo The VMCS info. object.
2122	*/
2123	static void hmR0VmxCheckHostEferMsr(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2124	{
2125	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2126
2127	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2128	{
2129	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2130	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2131	uint64_t uVmcsEferMsrVmcs;
2132	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2133	AssertRC(rc);
2134
2135	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2136	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2137	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2138	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2139	}
2140	}
2141
2142
2143	/**
2144	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2145	* VMCS are correct.
2146	*
2147	* @param pVCpu The cross context virtual CPU structure.
2148	* @param pVmcsInfo The VMCS info. object.
2149	*/
2150	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2151	{
2152	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2153
2154	/* Read the various MSR-area counts from the VMCS. */
2155	uint32_t cEntryLoadMsrs;
2156	uint32_t cExitStoreMsrs;
2157	uint32_t cExitLoadMsrs;
2158	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2159	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2160	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2161
2162	/* Verify all the MSR counts are the same. */
2163	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2164	Assert(cExitStoreMsrs == cExitLoadMsrs);
2165	uint32_t const cMsrs = cExitLoadMsrs;
2166
2167	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2168	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2169
2170	/* Verify the MSR counts are within the allocated page size. */
2171	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2172
2173	/* Verify the relevant contents of the MSR areas match. */
2174	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2175	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2176	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2177	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2178	for (uint32_t i = 0; i < cMsrs; i++)
2179	{
2180	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2181	if (fSeparateExitMsrStorePage)
2182	{
2183	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2184	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2185	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2186	}
2187
2188	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2189	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2190	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2191
2192	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2193	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2194	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2195	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2196
2197	/* Verify that cached host EFER MSR matches what's loaded the CPU. */
2198	bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2199	if (fIsEferMsr)
2200	{
2201	AssertMsgReturnVoid(u64Msr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2202	("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2203	pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64Msr, cMsrs));
2204	}
2205
2206	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2207	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2208	{
2209	uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2210	if (fIsEferMsr)
2211	{
2212	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2213	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2214	}
2215	else
2216	{
2217	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2218	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2219	}
2220	}
2221
2222	/* Move to the next MSR. */
2223	pHostMsrLoad++;
2224	pGuestMsrLoad++;
2225	pGuestMsrStore++;
2226	}
2227	}
2228	#endif /* VBOX_STRICT */
2229
2230
2231	/**
2232	* Flushes the TLB using EPT.
2233	*
2234	* @returns VBox status code.
2235	* @param pVCpu The cross context virtual CPU structure of the calling
2236	* EMT. Can be NULL depending on @a enmTlbFlush.
2237	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2238	* enmTlbFlush.
2239	* @param enmTlbFlush Type of flush.
2240	*
2241	* @remarks Caller is responsible for making sure this function is called only
2242	* when NestedPaging is supported and providing @a enmTlbFlush that is
2243	* supported by the CPU.
2244	* @remarks Can be called with interrupts disabled.
2245	*/
2246	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2247	{
2248	uint64_t au64Descriptor[2];
2249	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2250	au64Descriptor[0] = 0;
2251	else
2252	{
2253	Assert(pVCpu);
2254	Assert(pVmcsInfo);
2255	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2256	}
2257	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2258
2259	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2260	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2261
2262	if ( RT_SUCCESS(rc)
2263	&& pVCpu)
2264	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2265	}
2266
2267
2268	/**
2269	* Flushes the TLB using VPID.
2270	*
2271	* @returns VBox status code.
2272	* @param pVCpu The cross context virtual CPU structure of the calling
2273	* EMT. Can be NULL depending on @a enmTlbFlush.
2274	* @param enmTlbFlush Type of flush.
2275	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2276	* on @a enmTlbFlush).
2277	*
2278	* @remarks Can be called with interrupts disabled.
2279	*/
2280	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2281	{
2282	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2283
2284	uint64_t au64Descriptor[2];
2285	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2286	{
2287	au64Descriptor[0] = 0;
2288	au64Descriptor[1] = 0;
2289	}
2290	else
2291	{
2292	AssertPtr(pVCpu);
2293	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2294	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2295	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2296	au64Descriptor[1] = GCPtr;
2297	}
2298
2299	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2300	AssertMsg(rc == VINF_SUCCESS,
2301	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2302
2303	if ( RT_SUCCESS(rc)
2304	&& pVCpu)
2305	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2306	NOREF(rc);
2307	}
2308
2309
2310	/**
2311	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2312	* otherwise there is nothing really to invalidate.
2313	*
2314	* @returns VBox status code.
2315	* @param pVCpu The cross context virtual CPU structure.
2316	* @param GCVirt Guest virtual address of the page to invalidate.
2317	*/
2318	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2319	{
2320	AssertPtr(pVCpu);
2321	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2322
2323	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2324	{
2325	/*
2326	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2327	* the EPT case. See @bugref{6043} and @bugref{6177}.
2328	*
2329	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2330	* as this function maybe called in a loop with individual addresses.
2331	*/
2332	PVM pVM = pVCpu->CTX_SUFF(pVM);
2333	if (pVM->hm.s.vmx.fVpid)
2334	{
2335	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2336
2337	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2338	/*
2339	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2340	* where executing INVVPID outside 64-bit mode does not flush translations of
2341	* 64-bit linear addresses, see @bugref{6208#c72}.
2342	*/
2343	if (RT_HI_U32(GCVirt))
2344	fVpidFlush = false;
2345	#endif
2346
2347	if (fVpidFlush)
2348	{
2349	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2350	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2351	}
2352	else
2353	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2354	}
2355	else if (pVM->hm.s.fNestedPaging)
2356	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2357	}
2358
2359	return VINF_SUCCESS;
2360	}
2361
2362
2363	/**
2364	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2365	* case where neither EPT nor VPID is supported by the CPU.
2366	*
2367	* @param pHostCpu The HM physical-CPU structure.
2368	* @param pVCpu The cross context virtual CPU structure.
2369	*
2370	* @remarks Called with interrupts disabled.
2371	*/
2372	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2373	{
2374	AssertPtr(pVCpu);
2375	AssertPtr(pHostCpu);
2376
2377	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2378
2379	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2380	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2381	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2382	pVCpu->hm.s.fForceTLBFlush = false;
2383	return;
2384	}
2385
2386
2387	/**
2388	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2389	*
2390	* @param pHostCpu The HM physical-CPU structure.
2391	* @param pVCpu The cross context virtual CPU structure.
2392	* @param pVmcsInfo The VMCS info. object.
2393	*
2394	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2395	* nomenclature. The reason is, to avoid confusion in compare statements
2396	* since the host-CPU copies are named "ASID".
2397	*
2398	* @remarks Called with interrupts disabled.
2399	*/
2400	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2401	{
2402	#ifdef VBOX_WITH_STATISTICS
2403	bool fTlbFlushed = false;
2404	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2405	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2406	if (!fTlbFlushed) \
2407	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2408	} while (0)
2409	#else
2410	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2411	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2412	#endif
2413
2414	AssertPtr(pVCpu);
2415	AssertPtr(pHostCpu);
2416	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2417
2418	PVM pVM = pVCpu->CTX_SUFF(pVM);
2419	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2420	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2421	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2422
2423	/*
2424	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2425	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2426	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2427	* cannot reuse the current ASID anymore.
2428	*/
2429	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2430	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2431	{
2432	++pHostCpu->uCurrentAsid;
2433	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2434	{
2435	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2436	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2437	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2438	}
2439
2440	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2441	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2442	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2443
2444	/*
2445	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2446	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2447	*/
2448	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2449	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2450	HMVMX_SET_TAGGED_TLB_FLUSHED();
2451	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2452	}
2453	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2454	{
2455	/*
2456	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2457	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2458	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2459	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2460	* mappings, see @bugref{6568}.
2461	*
2462	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2463	*/
2464	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2465	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2466	HMVMX_SET_TAGGED_TLB_FLUSHED();
2467	}
2468
2469	pVCpu->hm.s.fForceTLBFlush = false;
2470	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2471
2472	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
2473	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
2474	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2475	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2476	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2477	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2478	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2479	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2480	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2481
2482	/* Update VMCS with the VPID. */
2483	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2484	AssertRC(rc);
2485
2486	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2487	}
2488
2489
2490	/**
2491	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2492	*
2493	* @param pHostCpu The HM physical-CPU structure.
2494	* @param pVCpu The cross context virtual CPU structure.
2495	* @param pVmcsInfo The VMCS info. object.
2496	*
2497	* @remarks Called with interrupts disabled.
2498	*/
2499	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2500	{
2501	AssertPtr(pVCpu);
2502	AssertPtr(pHostCpu);
2503	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2504	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2505	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2506
2507	/*
2508	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2509	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2510	*/
2511	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2512	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2513	{
2514	pVCpu->hm.s.fForceTLBFlush = true;
2515	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2516	}
2517
2518	/* Check for explicit TLB flushes. */
2519	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2520	{
2521	pVCpu->hm.s.fForceTLBFlush = true;
2522	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2523	}
2524
2525	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2526	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2527
2528	if (pVCpu->hm.s.fForceTLBFlush)
2529	{
2530	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2531	pVCpu->hm.s.fForceTLBFlush = false;
2532	}
2533	}
2534
2535
2536	/**
2537	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2538	*
2539	* @param pHostCpu The HM physical-CPU structure.
2540	* @param pVCpu The cross context virtual CPU structure.
2541	*
2542	* @remarks Called with interrupts disabled.
2543	*/
2544	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2545	{
2546	AssertPtr(pVCpu);
2547	AssertPtr(pHostCpu);
2548	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2549	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2550	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2551
2552	/*
2553	* Force a TLB flush for the first world switch if the current CPU differs from the one we
2554	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2555	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2556	* cannot reuse the current ASID anymore.
2557	*/
2558	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2559	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2560	{
2561	pVCpu->hm.s.fForceTLBFlush = true;
2562	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2563	}
2564
2565	/* Check for explicit TLB flushes. */
2566	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2567	{
2568	/*
2569	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2570	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2571	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
2572	* include fExplicitFlush's too) - an obscure corner case.
2573	*/
2574	pVCpu->hm.s.fForceTLBFlush = true;
2575	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2576	}
2577
2578	PVM pVM = pVCpu->CTX_SUFF(pVM);
2579	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2580	if (pVCpu->hm.s.fForceTLBFlush)
2581	{
2582	++pHostCpu->uCurrentAsid;
2583	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2584	{
2585	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2586	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2587	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2588	}
2589
2590	pVCpu->hm.s.fForceTLBFlush = false;
2591	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2592	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2593	if (pHostCpu->fFlushAsidBeforeUse)
2594	{
2595	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2596	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2597	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2598	{
2599	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2600	pHostCpu->fFlushAsidBeforeUse = false;
2601	}
2602	else
2603	{
2604	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2605	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2606	}
2607	}
2608	}
2609
2610	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2611	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2612	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2613	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2614	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2615	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2616	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2617
2618	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2619	AssertRC(rc);
2620	}
2621
2622
2623	/**
2624	* Flushes the guest TLB entry based on CPU capabilities.
2625	*
2626	* @param pHostCpu The HM physical-CPU structure.
2627	* @param pVCpu The cross context virtual CPU structure.
2628	* @param pVmcsInfo The VMCS info. object.
2629	*
2630	* @remarks Called with interrupts disabled.
2631	*/
2632	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2633	{
2634	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2635	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2636	#endif
2637	PVM pVM = pVCpu->CTX_SUFF(pVM);
2638	switch (pVM->hm.s.vmx.enmTlbFlushType)
2639	{
2640	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
2641	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
2642	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
2643	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
2644	default:
2645	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2646	break;
2647	}
2648	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2649	}
2650
2651
2652	/**
2653	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2654	* TLB entries from the host TLB before VM-entry.
2655	*
2656	* @returns VBox status code.
2657	* @param pVM The cross context VM structure.
2658	*/
2659	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2660	{
2661	/*
2662	* Determine optimal flush type for nested paging.
2663	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
2664	* unrestricted guest execution (see hmR3InitFinalizeR0()).
2665	*/
2666	if (pVM->hm.s.fNestedPaging)
2667	{
2668	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2669	{
2670	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2671	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2672	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2673	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2674	else
2675	{
2676	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2677	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2678	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2679	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2680	}
2681
2682	/* Make sure the write-back cacheable memory type for EPT is supported. */
2683	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2684	{
2685	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2686	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2687	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2688	}
2689
2690	/* EPT requires a page-walk length of 4. */
2691	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2692	{
2693	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2694	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2695	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2696	}
2697	}
2698	else
2699	{
2700	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2701	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2702	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2703	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2704	}
2705	}
2706
2707	/*
2708	* Determine optimal flush type for VPID.
2709	*/
2710	if (pVM->hm.s.vmx.fVpid)
2711	{
2712	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2713	{
2714	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2715	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2716	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2717	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2718	else
2719	{
2720	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2721	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2722	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2723	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2724	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2725	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2726	pVM->hm.s.vmx.fVpid = false;
2727	}
2728	}
2729	else
2730	{
2731	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2732	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2733	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2734	pVM->hm.s.vmx.fVpid = false;
2735	}
2736	}
2737
2738	/*
2739	* Setup the handler for flushing tagged-TLBs.
2740	*/
2741	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2742	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2743	else if (pVM->hm.s.fNestedPaging)
2744	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2745	else if (pVM->hm.s.vmx.fVpid)
2746	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2747	else
2748	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2749	return VINF_SUCCESS;
2750	}
2751
2752
2753	/**
2754	* Sets up the virtual-APIC page address for the VMCS.
2755	*
2756	* @returns VBox status code.
2757	* @param pVCpu The cross context virtual CPU structure.
2758	* @param pVmcsInfo The VMCS info. object.
2759	*/
2760	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2761	{
2762	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2763	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
2764	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
2765	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2766	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
2767	}
2768
2769
2770	/**
2771	* Sets up the MSR-bitmap address for the VMCS.
2772	*
2773	* @returns VBox status code.
2774	* @param pVCpu The cross context virtual CPU structure.
2775	* @param pVmcsInfo The VMCS info. object.
2776	*/
2777	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2778	{
2779	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2780	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
2781	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
2782	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2783	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
2784	}
2785
2786
2787	/**
2788	* Sets up the APIC-access page address for the VMCS.
2789	*
2790	* @returns VBox status code.
2791	* @param pVCpu The cross context virtual CPU structure.
2792	*/
2793	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
2794	{
2795	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
2796	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
2797	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2798	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
2799	}
2800
2801
2802	/**
2803	* Sets up the VMCS link pointer for the VMCS.
2804	*
2805	* @returns VBox status code.
2806	* @param pVCpu The cross context virtual CPU structure.
2807	* @param pVmcsInfo The VMCS info. object.
2808	*/
2809	DECLINLINE(int) hmR0VmxSetupVmcsLinkPtr(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2810	{
2811	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2812	uint64_t const u64VmcsLinkPtr = pVmcsInfo->u64VmcsLinkPtr;
2813	Assert(u64VmcsLinkPtr == UINT64_C(0xffffffffffffffff)); /* Bits 63:0 MB1. */
2814	return VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, u64VmcsLinkPtr);
2815	}
2816
2817
2818	/**
2819	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
2820	* in the VMCS.
2821	*
2822	* @returns VBox status code.
2823	* @param pVCpu The cross context virtual CPU structure.
2824	* @param pVmcsInfo The VMCS info. object.
2825	*/
2826	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2827	{
2828	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2829
2830	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
2831	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
2832	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2833
2834	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
2835	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
2836	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
2837
2838	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
2839	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
2840	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2841
2842	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
2843	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
2844	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
2845	AssertRCReturn(rc, rc);
2846	return VINF_SUCCESS;
2847	}
2848
2849
2850	/**
2851	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
2852	*
2853	* @param pVCpu The cross context virtual CPU structure.
2854	* @param pVmcsInfo The VMCS info. object.
2855	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2856	*/
2857	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2858	{
2859	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
2860
2861	/*
2862	* The guest can access the following MSRs (read, write) without causing
2863	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
2864	*/
2865	PVM pVM = pVCpu->CTX_SUFF(pVM);
2866	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
2867	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
2868	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
2869	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2870	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
2871
2872	/*
2873	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
2874	* associated with then. We never need to intercept access (writes need to be
2875	* executed without causing a VM-exit, reads will #GP fault anyway).
2876	*
2877	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
2878	* read/write them. We swap the the guest/host MSR value using the
2879	* auto-load/store MSR area.
2880	*/
2881	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2882	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
2883	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
2884	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
2885	if (pVM->cpum.ro.GuestFeatures.fIbrs)
2886	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
2887
2888	#if HC_ARCH_BITS == 64
2889	/*
2890	* Allow full read/write access for the following MSRs (mandatory for VT-x)
2891	* required for 64-bit guests.
2892	*/
2893	if (pVM->hm.s.fAllow64BitGuests)
2894	{
2895	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
2896	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
2897	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
2898	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2899	}
2900	#endif
2901
2902	/*
2903	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
2904	*/
2905	#ifdef VBOX_STRICT
2906	Assert(pVmcsInfo->pvMsrBitmap);
2907	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
2908	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
2909	#endif
2910	}
2911
2912
2913	/**
2914	* Sets up pin-based VM-execution controls in the VMCS.
2915	*
2916	* @returns VBox status code.
2917	* @param pVCpu The cross context virtual CPU structure.
2918	* @param pVmcsInfo The VMCS info. object.
2919	*/
2920	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2921	{
2922	PVM pVM = pVCpu->CTX_SUFF(pVM);
2923	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
2924	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2925
2926	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2927	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2928
2929	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2930	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2931
2932	/* Enable the VMX-preemption timer. */
2933	if (pVM->hm.s.vmx.fUsePreemptTimer)
2934	{
2935	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2936	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
2937	}
2938
2939	#if 0
2940	/* Enable posted-interrupt processing. */
2941	if (pVM->hm.s.fPostedIntrs)
2942	{
2943	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2944	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2945	fVal \|= VMX_PIN_CTL_POSTED_INT;
2946	}
2947	#endif
2948
2949	if ((fVal & fZap) != fVal)
2950	{
2951	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2952	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
2953	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2954	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2955	}
2956
2957	/* Commit it to the VMCS and update our cache. */
2958	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2959	AssertRCReturn(rc, rc);
2960	pVmcsInfo->u32PinCtls = fVal;
2961
2962	return VINF_SUCCESS;
2963	}
2964
2965
2966	/**
2967	* Sets up secondary processor-based VM-execution controls in the VMCS.
2968	*
2969	* @returns VBox status code.
2970	* @param pVCpu The cross context virtual CPU structure.
2971	* @param pVmcsInfo The VMCS info. object.
2972	*/
2973	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2974	{
2975	PVM pVM = pVCpu->CTX_SUFF(pVM);
2976	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
2977	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2978
2979	/* WBINVD causes a VM-exit. */
2980	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2981	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
2982
2983	/* Enable EPT (aka nested-paging). */
2984	if (pVM->hm.s.fNestedPaging)
2985	fVal \|= VMX_PROC_CTLS2_EPT;
2986
2987	/* Enable the INVPCID instruction if supported by the hardware and we expose
2988	it to the guest. Without this, guest executing INVPCID would cause a #UD. */
2989	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2990	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
2991	fVal \|= VMX_PROC_CTLS2_INVPCID;
2992
2993	/* Enable VPID. */
2994	if (pVM->hm.s.vmx.fVpid)
2995	fVal \|= VMX_PROC_CTLS2_VPID;
2996
2997	/* Enable unrestricted guest execution. */
2998	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2999	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3000
3001	#if 0
3002	if (pVM->hm.s.fVirtApicRegs)
3003	{
3004	/* Enable APIC-register virtualization. */
3005	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3006	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3007
3008	/* Enable virtual-interrupt delivery. */
3009	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3010	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3011	}
3012	#endif
3013
3014	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
3015	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3016	* done dynamically. */
3017	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3018	{
3019	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3020	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3021	AssertRCReturn(rc, rc);
3022	}
3023
3024	/* Enable the RDTSCP instruction if supported by the hardware and we expose
3025	it to the guest. Without this, guest executing RDTSCP would cause a #UD. */
3026	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
3027	&& pVM->cpum.ro.GuestFeatures.fRdTscP)
3028	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3029
3030	/* Enable Pause-Loop exiting. */
3031	if ( pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT
3032	&& pVM->hm.s.vmx.cPleGapTicks
3033	&& pVM->hm.s.vmx.cPleWindowTicks)
3034	{
3035	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3036
3037	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3038	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3039	AssertRCReturn(rc, rc);
3040	}
3041
3042	if ((fVal & fZap) != fVal)
3043	{
3044	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3045	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3046	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3047	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3048	}
3049
3050	/* Commit it to the VMCS and update our cache. */
3051	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3052	AssertRCReturn(rc, rc);
3053	pVmcsInfo->u32ProcCtls2 = fVal;
3054
3055	return VINF_SUCCESS;
3056	}
3057
3058
3059	/**
3060	* Sets up processor-based VM-execution controls in the VMCS.
3061	*
3062	* @returns VBox status code.
3063	* @param pVCpu The cross context virtual CPU structure.
3064	* @param pVmcsInfo The VMCS info. object.
3065	*/
3066	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3067	{
3068	PVM pVM = pVCpu->CTX_SUFF(pVM);
3069
3070	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3071	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3072
3073	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3074	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3075	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3076	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3077	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3078	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3079	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3080
3081	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3082	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3083	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3084	{
3085	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3086	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3087	}
3088
3089	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3090	if (!pVM->hm.s.fNestedPaging)
3091	{
3092	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3093	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3094	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3095	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3096	}
3097
3098	/* Use TPR shadowing if supported by the CPU. */
3099	if ( PDMHasApic(pVM)
3100	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3101	{
3102	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3103	/* CR8 writes cause a VM-exit based on TPR threshold. */
3104	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3105	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3106	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3107	AssertRCReturn(rc, rc);
3108	}
3109	else
3110	{
3111	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3112	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3113	if (pVM->hm.s.fAllow64BitGuests)
3114	{
3115	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3116	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3117	}
3118	}
3119
3120	/* Use MSR-bitmaps if supported by the CPU. */
3121	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3122	{
3123	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3124	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3125	AssertRCReturn(rc, rc);
3126	}
3127
3128	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3129	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3130	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3131
3132	if ((fVal & fZap) != fVal)
3133	{
3134	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3135	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3136	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3137	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3138	}
3139
3140	/* Commit it to the VMCS and update our cache. */
3141	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3142	AssertRCReturn(rc, rc);
3143	pVmcsInfo->u32ProcCtls = fVal;
3144
3145	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3146	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3147	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo, false /* fIsNstGstVmcs */);
3148
3149	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3150	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3151	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3152
3153	/* Sanity check, should not really happen. */
3154	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3155	{ /* likely */ }
3156	else
3157	{
3158	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3159	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3160	}
3161
3162	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3163	return VINF_SUCCESS;
3164	}
3165
3166
3167	/**
3168	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3169	* Processor-based VM-execution) control fields in the VMCS.
3170	*
3171	* @returns VBox status code.
3172	* @param pVCpu The cross context virtual CPU structure.
3173	* @param pVmcsInfo The VMCS info. object.
3174	*/
3175	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3176	{
3177	/* Set the auto-load/store MSR area addresses in the VMCS. */
3178	int rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3179	if (RT_SUCCESS(rc))
3180	{
3181	/* Set the VMCS link pointer in the VMCS. */
3182	rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3183	if (RT_SUCCESS(rc))
3184	{
3185	/* Set the CR0/CR4 guest/host mask. */
3186	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3187	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3188	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3189	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3190	if (RT_SUCCESS(rc))
3191	{
3192	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3193	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3194	return VINF_SUCCESS;
3195	}
3196	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3197	}
3198	else
3199	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3200	}
3201	else
3202	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3203	return rc;
3204	}
3205
3206
3207	/**
3208	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3209	*
3210	* We shall setup those exception intercepts that don't change during the
3211	* lifetime of the VM here. The rest are done dynamically while loading the
3212	* guest state.
3213	*
3214	* @returns VBox status code.
3215	* @param pVCpu The cross context virtual CPU structure.
3216	* @param pVmcsInfo The VMCS info. object.
3217	*/
3218	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3219	{
3220	/*
3221	* The following exceptions are always intercepted:
3222	*
3223	* #AC - To prevent the guest from hanging the CPU.
3224	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3225	* recursive #DBs can cause a CPU hang.
3226	* #PF - To sync our shadow page tables when nested-paging is not used.
3227	*/
3228	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3229	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3230	\| RT_BIT(X86_XCPT_DB)
3231	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3232
3233	/* Commit it to the VMCS. */
3234	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3235	AssertRCReturn(rc, rc);
3236
3237	/* Update our cache of the exception bitmap. */
3238	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3239	return VINF_SUCCESS;
3240	}
3241
3242
3243	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3244	/**
3245	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3246	*
3247	* @returns VBox status code.
3248	* @param pVCpu The cross context virtual CPU structure.
3249	* @param pVmcsInfo The VMCS info. object.
3250	*/
3251	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3252	{
3253	PVM pVM = pVCpu->CTX_SUFF(pVM);
3254	int rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3255	if (RT_SUCCESS(rc))
3256	{
3257	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3258	if (RT_SUCCESS(rc))
3259	{
3260	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3261	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3262	if (RT_SUCCESS(rc))
3263	{
3264	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3265	rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3266	if (RT_SUCCESS(rc))
3267	return VINF_SUCCESS;
3268
3269	LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3270	}
3271	else
3272	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3273	}
3274	else
3275	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3276	}
3277	else
3278	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3279
3280	return rc;
3281	}
3282	#endif
3283
3284
3285	/**
3286	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3287	* VMX.
3288	*
3289	* @returns VBox status code.
3290	* @param pVCpu The cross context virtual CPU structure.
3291	* @param pVmcsInfo The VMCS info. object.
3292	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3293	*/
3294	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3295	{
3296	Assert(pVmcsInfo);
3297	Assert(pVmcsInfo->pvVmcs);
3298	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3299
3300	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3301	PVM pVM = pVCpu->CTX_SUFF(pVM);
3302	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3303	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3304
3305	LogFlowFunc(("\n"));
3306
3307	/*
3308	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3309	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3310	*/
3311	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3312	if (RT_SUCCESS(rc))
3313	{
3314	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3315	if (RT_SUCCESS(rc))
3316	{
3317	if (!fIsNstGstVmcs)
3318	{
3319	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3320	if (RT_SUCCESS(rc))
3321	{
3322	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
3323	if (RT_SUCCESS(rc))
3324	{
3325	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
3326	if (RT_SUCCESS(rc))
3327	{
3328	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
3329	if (RT_SUCCESS(rc))
3330	{ /* likely */ }
3331	else
3332	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
3333	}
3334	else
3335	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
3336	}
3337	else
3338	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
3339	}
3340	else
3341	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
3342	}
3343	else
3344	{
3345	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3346	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
3347	if (RT_SUCCESS(rc))
3348	{ /* likely */ }
3349	else
3350	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
3351	#else
3352	AssertFailed();
3353	#endif
3354	}
3355	}
3356	else
3357	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
3358	}
3359	else
3360	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
3361
3362	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
3363	if (RT_SUCCESS(rc))
3364	{
3365	rc = hmR0VmxClearVmcs(pVmcsInfo);
3366	if (RT_SUCCESS(rc))
3367	{ /* likely */ }
3368	else
3369	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
3370	}
3371
3372	/*
3373	* Update the last-error record both for failures and success, so we
3374	* can propagate the status code back to ring-3 for diagnostics.
3375	*/
3376	hmR0VmxUpdateErrorRecord(pVCpu, rc);
3377	NOREF(pszVmcs);
3378	return rc;
3379	}
3380
3381
3382	/**
3383	* Does global VT-x initialization (called during module initialization).
3384	*
3385	* @returns VBox status code.
3386	*/
3387	VMMR0DECL(int) VMXR0GlobalInit(void)
3388	{
3389	#ifdef HMVMX_USE_FUNCTION_TABLE
3390	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
3391	# ifdef VBOX_STRICT
3392	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
3393	Assert(g_apfnVMExitHandlers[i]);
3394	# endif
3395	#endif
3396	return VINF_SUCCESS;
3397	}
3398
3399
3400	/**
3401	* Does global VT-x termination (called during module termination).
3402	*/
3403	VMMR0DECL(void) VMXR0GlobalTerm()
3404	{
3405	/* Nothing to do currently. */
3406	}
3407
3408
3409	/**
3410	* Sets up and activates VT-x on the current CPU.
3411	*
3412	* @returns VBox status code.
3413	* @param pHostCpu The HM physical-CPU structure.
3414	* @param pVM The cross context VM structure. Can be
3415	* NULL after a host resume operation.
3416	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
3417	* fEnabledByHost is @c true).
3418	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
3419	* @a fEnabledByHost is @c true).
3420	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
3421	* enable VT-x on the host.
3422	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
3423	*/
3424	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
3425	PCSUPHWVIRTMSRS pHwvirtMsrs)
3426	{
3427	Assert(pHostCpu);
3428	Assert(pHwvirtMsrs);
3429	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3430
3431	/* Enable VT-x if it's not already enabled by the host. */
3432	if (!fEnabledByHost)
3433	{
3434	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
3435	if (RT_FAILURE(rc))
3436	return rc;
3437	}
3438
3439	/*
3440	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
3441	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
3442	* invalidated when flushing by VPID.
3443	*/
3444	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3445	{
3446	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
3447	pHostCpu->fFlushAsidBeforeUse = false;
3448	}
3449	else
3450	pHostCpu->fFlushAsidBeforeUse = true;
3451
3452	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
3453	++pHostCpu->cTlbFlushes;
3454
3455	return VINF_SUCCESS;
3456	}
3457
3458
3459	/**
3460	* Deactivates VT-x on the current CPU.
3461	*
3462	* @returns VBox status code.
3463	* @param pvCpuPage Pointer to the VMXON region.
3464	* @param HCPhysCpuPage Physical address of the VMXON region.
3465	*
3466	* @remarks This function should never be called when SUPR0EnableVTx() or
3467	* similar was used to enable VT-x on the host.
3468	*/
3469	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
3470	{
3471	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
3472
3473	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3474	return hmR0VmxLeaveRootMode();
3475	}
3476
3477
3478	/**
3479	* Does per-VM VT-x initialization.
3480	*
3481	* @returns VBox status code.
3482	* @param pVM The cross context VM structure.
3483	*/
3484	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
3485	{
3486	LogFlowFunc(("pVM=%p\n", pVM));
3487
3488	int rc = hmR0VmxStructsAlloc(pVM);
3489	if (RT_FAILURE(rc))
3490	{
3491	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
3492	return rc;
3493	}
3494
3495	return VINF_SUCCESS;
3496	}
3497
3498
3499	/**
3500	* Does per-VM VT-x termination.
3501	*
3502	* @returns VBox status code.
3503	* @param pVM The cross context VM structure.
3504	*/
3505	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
3506	{
3507	LogFlowFunc(("pVM=%p\n", pVM));
3508
3509	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3510	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
3511	{
3512	Assert(pVM->hm.s.vmx.pvScratch);
3513	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
3514	}
3515	#endif
3516	hmR0VmxStructsFree(pVM);
3517	return VINF_SUCCESS;
3518	}
3519
3520
3521	/**
3522	* Sets up the VM for execution using hardware-assisted VMX.
3523	* This function is only called once per-VM during initialization.
3524	*
3525	* @returns VBox status code.
3526	* @param pVM The cross context VM structure.
3527	*/
3528	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
3529	{
3530	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
3531	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3532
3533	LogFlowFunc(("pVM=%p\n", pVM));
3534
3535	/*
3536	* At least verify if VMX is enabled, since we can't check if we're in
3537	* VMX root mode or not without causing a #GP.
3538	*/
3539	RTCCUINTREG const uHostCR4 = ASMGetCR4();
3540	if (RT_LIKELY(uHostCR4 & X86_CR4_VMXE))
3541	{ /* likely */ }
3542	else
3543	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
3544
3545	/*
3546	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
3547	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
3548	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
3549	*/
3550	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3551	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
3552	\|\| !pVM->hm.s.vmx.pRealModeTSS))
3553	{
3554	LogRelFunc(("Invalid real-on-v86 state.\n"));
3555	return VERR_INTERNAL_ERROR;
3556	}
3557
3558	/* Initialize these always, see hmR3InitFinalizeR0().*/
3559	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
3560	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
3561
3562	/* Setup the tagged-TLB flush handlers. */
3563	int rc = hmR0VmxSetupTaggedTlb(pVM);
3564	if (RT_FAILURE(rc))
3565	{
3566	LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
3567	return rc;
3568	}
3569
3570	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
3571	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
3572	#if HC_ARCH_BITS == 64
3573	if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
3574	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
3575	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
3576	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
3577	#endif
3578
3579	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3580	{
3581	PVMCPU pVCpu = &pVM->aCpus[idCpu];
3582	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
3583
3584	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
3585	if (RT_SUCCESS(rc))
3586	{
3587	#if HC_ARCH_BITS == 32
3588	hmR0VmxInitVmcsReadCache(pVCpu);
3589	#endif
3590	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3591	if (pVM->cpum.ro.GuestFeatures.fVmx)
3592	{
3593	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
3594	if (RT_SUCCESS(rc))
3595	{ /* likely */ }
3596	else
3597	{
3598	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
3599	return rc;
3600	}
3601	}
3602	#endif
3603	}
3604	else
3605	{
3606	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
3607	return rc;
3608	}
3609	}
3610
3611	return VINF_SUCCESS;
3612	}
3613
3614
3615	#if HC_ARCH_BITS == 32
3616	# ifdef VBOX_ENABLE_64_BITS_GUESTS
3617	/**
3618	* Check if guest state allows safe use of 32-bit switcher again.
3619	*
3620	* Segment bases and protected mode structures must be 32-bit addressable
3621	* because the 32-bit switcher will ignore high dword when writing these VMCS
3622	* fields. See @bugref{8432} for details.
3623	*
3624	* @returns true if safe, false if must continue to use the 64-bit switcher.
3625	* @param pCtx Pointer to the guest-CPU context.
3626	*
3627	* @remarks No-long-jump zone!!!
3628	*/
3629	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
3630	{
3631	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
3632	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
3633	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3634	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3635	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
3636	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3637	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
3638	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
3639	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3640	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3641
3642	/* All good, bases are 32-bit. */
3643	return true;
3644	}
3645	# endif /* VBOX_ENABLE_64_BITS_GUESTS */
3646
3647	# ifdef VBOX_STRICT
3648	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
3649	{
3650	switch (idxField)
3651	{
3652	case VMX_VMCS_GUEST_RIP:
3653	case VMX_VMCS_GUEST_RSP:
3654	case VMX_VMCS_GUEST_SYSENTER_EIP:
3655	case VMX_VMCS_GUEST_SYSENTER_ESP:
3656	case VMX_VMCS_GUEST_GDTR_BASE:
3657	case VMX_VMCS_GUEST_IDTR_BASE:
3658	case VMX_VMCS_GUEST_CS_BASE:
3659	case VMX_VMCS_GUEST_DS_BASE:
3660	case VMX_VMCS_GUEST_ES_BASE:
3661	case VMX_VMCS_GUEST_FS_BASE:
3662	case VMX_VMCS_GUEST_GS_BASE:
3663	case VMX_VMCS_GUEST_SS_BASE:
3664	case VMX_VMCS_GUEST_LDTR_BASE:
3665	case VMX_VMCS_GUEST_TR_BASE:
3666	case VMX_VMCS_GUEST_CR3:
3667	return true;
3668	}
3669	return false;
3670	}
3671
3672	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3673	{
3674	switch (idxField)
3675	{
3676	/* Read-only fields. */
3677	case VMX_VMCS_RO_EXIT_QUALIFICATION:
3678	return true;
3679	}
3680	/* Remaining readable fields should also be writable. */
3681	return hmR0VmxIsValidWriteField(idxField);
3682	}
3683	# endif /* VBOX_STRICT */
3684
3685
3686	/**
3687	* Executes the specified handler in 64-bit mode.
3688	*
3689	* @returns VBox status code (no informational status codes).
3690	* @param pVCpu The cross context virtual CPU structure.
3691	* @param enmOp The operation to perform.
3692	* @param cParams Number of parameters.
3693	* @param paParam Array of 32-bit parameters.
3694	*/
3695	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
3696	{
3697	PVM pVM = pVCpu->CTX_SUFF(pVM);
3698	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
3699	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
3700	Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
3701	Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
3702
3703	#ifdef VBOX_STRICT
3704	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
3705	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
3706
3707	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
3708	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
3709	#endif
3710
3711	/* Disable interrupts. */
3712	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
3713
3714	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
3715	RTCPUID idHostCpu = RTMpCpuId();
3716	CPUMR0SetLApic(pVCpu, idHostCpu);
3717	#endif
3718
3719	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
3720
3721	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3722	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3723
3724	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
3725	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3726	hmR0VmxClearVmcs(pVmcsInfo);
3727
3728	/* Leave VMX root mode and disable VMX. */
3729	VMXDisable();
3730	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3731
3732	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
3733	CPUMSetHyperEIP(pVCpu, enmOp);
3734	for (int i = (int)cParams - 1; i >= 0; i--)
3735	CPUMPushHyper(pVCpu, paParam[i]);
3736
3737	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
3738
3739	/* Call the switcher. */
3740	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
3741	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
3742
3743	/* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
3744	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
3745
3746	/* Re-enter VMX root mode. */
3747	int rc2 = VMXEnable(HCPhysCpuPage);
3748	if (RT_FAILURE(rc2))
3749	{
3750	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3751	ASMSetFlags(fOldEFlags);
3752	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
3753	return rc2;
3754	}
3755
3756	/* Restore the VMCS as the current VMCS. */
3757	rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
3758	AssertRC(rc2);
3759	Assert(!(ASMGetFlags() & X86_EFL_IF));
3760	ASMSetFlags(fOldEFlags);
3761	return rc;
3762	}
3763
3764
3765	/**
3766	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
3767	* supporting 64-bit guests.
3768	*
3769	* @returns VBox status code.
3770	* @param fResume Whether to VMLAUNCH or VMRESUME.
3771	* @param pCtx Pointer to the guest-CPU context.
3772	* @param pCache Pointer to the VMCS batch cache.
3773	* @param pVM The cross context VM structure.
3774	* @param pVCpu The cross context virtual CPU structure.
3775	*/
3776	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
3777	{
3778	NOREF(fResume);
3779
3780	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3781	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3782	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3783
3784	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3785	pCache->uPos = 1;
3786	pCache->interPD = PGMGetInterPaeCR3(pVM);
3787	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
3788	#endif
3789
3790	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3791	pCache->TestIn.HCPhysCpuPage = 0;
3792	pCache->TestIn.HCPhysVmcs = 0;
3793	pCache->TestIn.pCache = 0;
3794	pCache->TestOut.HCPhysVmcs = 0;
3795	pCache->TestOut.pCache = 0;
3796	pCache->TestOut.pCtx = 0;
3797	pCache->TestOut.eflags = 0;
3798	#else
3799	NOREF(pCache);
3800	#endif
3801
3802	uint32_t aParam[10];
3803	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
3804	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
3805	aParam[2] = RT_LO_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
3806	aParam[3] = RT_HI_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
3807	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
3808	aParam[5] = 0;
3809	aParam[6] = VM_RC_ADDR(pVM, pVM);
3810	aParam[7] = 0;
3811	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
3812	aParam[9] = 0;
3813
3814	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3815	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
3816	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
3817	#endif
3818	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
3819
3820	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3821	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
3822	Assert(pCtx->dr[4] == 10);
3823	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
3824	#endif
3825
3826	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3827	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
3828	AssertMsg(pCache->TestIn.HCPhysVmcs == pVmcsInfo->HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3829	pVmcsInfo->HCPhysVmcs));
3830	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3831	pCache->TestOut.HCPhysVmcs));
3832	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
3833	pCache->TestOut.pCache));
3834	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
3835	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
3836	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
3837	pCache->TestOut.pCtx));
3838	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
3839	#endif
3840	NOREF(pCtx);
3841	return rc;
3842	}
3843	#endif
3844
3845
3846	/**
3847	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
3848	* the VMCS.
3849	*
3850	* @returns VBox status code.
3851	*/
3852	static int hmR0VmxExportHostControlRegs(void)
3853	{
3854	RTCCUINTREG uReg = ASMGetCR0();
3855	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
3856	AssertRCReturn(rc, rc);
3857
3858	uReg = ASMGetCR3();
3859	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
3860	AssertRCReturn(rc, rc);
3861
3862	uReg = ASMGetCR4();
3863	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
3864	AssertRCReturn(rc, rc);
3865	return rc;
3866	}
3867
3868
3869	/**
3870	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
3871	* the host-state area in the VMCS.
3872	*
3873	* @returns VBox status code.
3874	* @param pVCpu The cross context virtual CPU structure.
3875	*/
3876	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
3877	{
3878	#if HC_ARCH_BITS == 64
3879	/**
3880	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
3881	* requirements. See hmR0VmxExportHostSegmentRegs().
3882	*/
3883	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
3884	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
3885	{ \
3886	bool fValidSelector = true; \
3887	if ((selValue) & X86_SEL_LDT) \
3888	{ \
3889	uint32_t uAttr = ASMGetSegAttr((selValue)); \
3890	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
3891	} \
3892	if (fValidSelector) \
3893	{ \
3894	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
3895	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
3896	} \
3897	(selValue) = 0; \
3898	}
3899
3900	/*
3901	* If we've executed guest code using hardware-assisted VMX, the host-state bits
3902	* will be messed up. We should -not- save the messed up state without restoring
3903	* the original host-state, see @bugref{7240}.
3904	*
3905	* This apparently can happen (most likely the FPU changes), deal with it rather than
3906	* asserting. Was observed booting Solaris 10u10 32-bit guest.
3907	*/
3908	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
3909	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
3910	{
3911	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
3912	pVCpu->idCpu));
3913	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
3914	}
3915	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
3916	#else
3917	RT_NOREF(pVCpu);
3918	#endif
3919
3920	/*
3921	* Host DS, ES, FS and GS segment registers.
3922	*/
3923	#if HC_ARCH_BITS == 64
3924	RTSEL uSelDS = ASMGetDS();
3925	RTSEL uSelES = ASMGetES();
3926	RTSEL uSelFS = ASMGetFS();
3927	RTSEL uSelGS = ASMGetGS();
3928	#else
3929	RTSEL uSelDS = 0;
3930	RTSEL uSelES = 0;
3931	RTSEL uSelFS = 0;
3932	RTSEL uSelGS = 0;
3933	#endif
3934
3935	/*
3936	* Host CS and SS segment registers.
3937	*/
3938	RTSEL uSelCS = ASMGetCS();
3939	RTSEL uSelSS = ASMGetSS();
3940
3941	/*
3942	* Host TR segment register.
3943	*/
3944	RTSEL uSelTR = ASMGetTR();
3945
3946	#if HC_ARCH_BITS == 64
3947	/*
3948	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
3949	* gain VM-entry and restore them before we get preempted.
3950	*
3951	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
3952	*/
3953	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3954	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3955	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3956	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3957	# undef VMXLOCAL_ADJUST_HOST_SEG
3958	#endif
3959
3960	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
3961	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
3962	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
3963	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
3964	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
3965	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
3966	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
3967	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
3968	Assert(uSelCS);
3969	Assert(uSelTR);
3970
3971	/* Write these host selector fields into the host-state area in the VMCS. */
3972	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3973	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3974	#if HC_ARCH_BITS == 64
3975	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3976	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3977	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3978	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3979	#else
3980	NOREF(uSelDS);
3981	NOREF(uSelES);
3982	NOREF(uSelFS);
3983	NOREF(uSelGS);
3984	#endif
3985	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3986	AssertRCReturn(rc, rc);
3987
3988	/*
3989	* Host GDTR and IDTR.
3990	*/
3991	RTGDTR Gdtr;
3992	RTIDTR Idtr;
3993	RT_ZERO(Gdtr);
3994	RT_ZERO(Idtr);
3995	ASMGetGDTR(&Gdtr);
3996	ASMGetIDTR(&Idtr);
3997	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3998	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3999	AssertRCReturn(rc, rc);
4000
4001	#if HC_ARCH_BITS == 64
4002	/*
4003	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4004	* them to the maximum limit (0xffff) on every VM-exit.
4005	*/
4006	if (Gdtr.cbGdt != 0xffff)
4007	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4008
4009	/*
4010	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4011	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4012	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4013	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4014	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4015	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4016	* at 0xffff on hosts where we are sure it won't cause trouble.
4017	*/
4018	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4019	if (Idtr.cbIdt < 0x0fff)
4020	# else
4021	if (Idtr.cbIdt != 0xffff)
4022	# endif
4023	{
4024	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4025	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4026	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4027	}
4028	#endif
4029
4030	/*
4031	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4032	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4033	* RPL should be too in most cases.
4034	*/
4035	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4036	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4037
4038	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4039	#if HC_ARCH_BITS == 64
4040	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4041
4042	/*
4043	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4044	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4045	* restoration if the host has something else. Task switching is not supported in 64-bit
4046	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4047	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4048	*
4049	* [1] See Intel spec. 3.5 "System Descriptor Types".
4050	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4051	*/
4052	PVM pVM = pVCpu->CTX_SUFF(pVM);
4053	Assert(pDesc->System.u4Type == 11);
4054	if ( pDesc->System.u16LimitLow != 0x67
4055	\|\| pDesc->System.u4LimitHigh)
4056	{
4057	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4058	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4059	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4060	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4061	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4062	}
4063
4064	/*
4065	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4066	*/
4067	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4068	{
4069	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4070	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4071	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4072	{
4073	/* The GDT is read-only but the writable GDT is available. */
4074	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4075	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4076	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4077	AssertRCReturn(rc, rc);
4078	}
4079	}
4080	#else
4081	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4082	#endif
4083	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4084	AssertRCReturn(rc, rc);
4085
4086	/*
4087	* Host FS base and GS base.
4088	*/
4089	#if HC_ARCH_BITS == 64
4090	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4091	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4092	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4093	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4094	AssertRCReturn(rc, rc);
4095
4096	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4097	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4098	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4099	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4100	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4101	#endif
4102	return VINF_SUCCESS;
4103	}
4104
4105
4106	/**
4107	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4108	* host-state area of the VMCS.
4109	*
4110	* These MSRs will be automatically restored on the host after every successful
4111	* VM-exit.
4112	*
4113	* @returns VBox status code.
4114	* @param pVCpu The cross context virtual CPU structure.
4115	*
4116	* @remarks No-long-jump zone!!!
4117	*/
4118	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4119	{
4120	AssertPtr(pVCpu);
4121
4122	/*
4123	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4124	* rather than swapping them on every VM-entry.
4125	*/
4126	hmR0VmxLazySaveHostMsrs(pVCpu);
4127
4128	/*
4129	* Host Sysenter MSRs.
4130	*/
4131	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4132	#if HC_ARCH_BITS == 32
4133	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4134	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4135	#else
4136	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4137	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4138	#endif
4139	AssertRCReturn(rc, rc);
4140
4141	/*
4142	* Host EFER MSR.
4143	*
4144	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4145	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4146	*/
4147	PVM pVM = pVCpu->CTX_SUFF(pVM);
4148	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4149	{
4150	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4151	AssertRCReturn(rc, rc);
4152	}
4153
4154	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4155	* hmR0VmxExportGuestEntryExitCtls(). */
4156
4157	return VINF_SUCCESS;
4158	}
4159
4160
4161	/**
4162	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4163	*
4164	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4165	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4166	*
4167	* @returns true if we need to load guest EFER, false otherwise.
4168	* @param pVCpu The cross context virtual CPU structure.
4169	*
4170	* @remarks Requires EFER, CR4.
4171	* @remarks No-long-jump zone!!!
4172	*/
4173	static bool hmR0VmxShouldSwapEferMsr(PCVMCPU pVCpu)
4174	{
4175	#ifdef HMVMX_ALWAYS_SWAP_EFER
4176	RT_NOREF(pVCpu);
4177	return true;
4178	#else
4179	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4180	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4181	/* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4182	if (CPUMIsGuestInLongModeEx(pCtx))
4183	return false;
4184	#endif
4185
4186	PVM pVM = pVCpu->CTX_SUFF(pVM);
4187	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4188	uint64_t const u64GuestEfer = pCtx->msrEFER;
4189
4190	/*
4191	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4192	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4193	*/
4194	if ( CPUMIsGuestInLongModeEx(pCtx)
4195	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4196	return true;
4197
4198	/*
4199	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4200	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4201	*
4202	* See Intel spec. 4.5 "IA-32e Paging".
4203	* See Intel spec. 4.1.1 "Three Paging Modes".
4204	*
4205	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4206	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4207	*/
4208	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4209	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4210	if ( (pCtx->cr4 & X86_CR4_PAE)
4211	&& (pCtx->cr0 & X86_CR0_PG)
4212	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4213	{
4214	/* Assert that host is NX capable. */
4215	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4216	return true;
4217	}
4218
4219	return false;
4220	#endif
4221	}
4222
4223	/**
4224	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4225	* VMCS.
4226	*
4227	* This is typically required when the guest changes paging mode.
4228	*
4229	* @returns VBox status code.
4230	* @param pVCpu The cross context virtual CPU structure.
4231	* @param pVmxTransient The VMX-transient structure.
4232	*
4233	* @remarks Requires EFER.
4234	* @remarks No-long-jump zone!!!
4235	*/
4236	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4237	{
4238	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4239	{
4240	PVM pVM = pVCpu->CTX_SUFF(pVM);
4241	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4242
4243	/*
4244	* VM-entry controls.
4245	*/
4246	{
4247	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4248	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4249
4250	/*
4251	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4252	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4253	*
4254	* For nested-guests, this is a mandatory VM-entry control. It's also
4255	* required because we do not want to leak host bits to the nested-guest.
4256	*/
4257	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4258
4259	/*
4260	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4261	*
4262	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4263	* required to get the nested-guest working with hardware-assisted VMX execution.
4264	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested-hypervisor
4265	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4266	* here rather than while merging the guest VMCS controls.
4267	*/
4268	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4269	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4270	else
4271	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4272
4273	/*
4274	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4275	*
4276	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4277	* regardless of whether the nested-guest VMCS specifies it because we are free to
4278	* load whatever MSRs we require and we do not need to modify the guest visible copy
4279	* of the VM-entry MSR load area.
4280	*/
4281	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4282	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4283	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4284	else
4285	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4286
4287	/*
4288	* The following should -not- be set (since we're not in SMM mode):
4289	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4290	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4291	*/
4292
4293	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4294	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4295
4296	if ((fVal & fZap) == fVal)
4297	{ /* likely */ }
4298	else
4299	{
4300	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4301	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4302	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4303	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4304	}
4305
4306	/* Commit it to the VMCS. */
4307	if (pVmcsInfo->u32EntryCtls != fVal)
4308	{
4309	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4310	AssertRCReturn(rc, rc);
4311	pVmcsInfo->u32EntryCtls = fVal;
4312	}
4313	}
4314
4315	/*
4316	* VM-exit controls.
4317	*/
4318	{
4319	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4320	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4321
4322	/*
4323	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4324	* supported the 1-setting of this bit.
4325	*
4326	* For nested-guests, we set the "save debug controls" as the converse
4327	* "load debug controls" is mandatory for nested-guests anyway.
4328	*/
4329	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4330
4331	/*
4332	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4333	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4334	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4335	* hmR0VmxExportHostMsrs().
4336	*
4337	* For nested-guests, we always set this bit as we do not support 32-bit
4338	* hosts.
4339	*/
4340	#if HC_ARCH_BITS == 64
4341	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4342	#else
4343	Assert(!pVmxTransient->fIsNestedGuest);
4344	Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
4345	\|\| pVmcsInfo->pfnStartVM == VMXR0StartVM32);
4346	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
4347	if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
4348	{
4349	/* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
4350	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4351	}
4352	else
4353	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
4354	#endif
4355
4356	/*
4357	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4358	*
4359	* For nested-guests, we should use the "save IA32_EFER" control if we also
4360	* used the "load IA32_EFER" control while exporting VM-entry controls.
4361	*/
4362	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4363	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4364	{
4365	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4366	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4367	}
4368
4369	/*
4370	* Enable saving of the VMX-preemption timer value on VM-exit.
4371	* For nested-guests, currently not exposed/used.
4372	*/
4373	if ( pVM->hm.s.vmx.fUsePreemptTimer
4374	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4375	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4376
4377	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4378	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4379
4380	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4381	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4382	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4383
4384	if ((fVal & fZap) == fVal)
4385	{ /* likely */ }
4386	else
4387	{
4388	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4389	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4390	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4391	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4392	}
4393
4394	/* Commit it to the VMCS. */
4395	if (pVmcsInfo->u32ExitCtls != fVal)
4396	{
4397	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4398	AssertRCReturn(rc, rc);
4399	pVmcsInfo->u32ExitCtls = fVal;
4400	}
4401	}
4402
4403	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4404	}
4405	return VINF_SUCCESS;
4406	}
4407
4408
4409	/**
4410	* Sets the TPR threshold in the VMCS.
4411	*
4412	* @returns VBox status code.
4413	* @param pVCpu The cross context virtual CPU structure.
4414	* @param pVmcsInfo The VMCS info. object.
4415	* @param u32TprThreshold The TPR threshold (task-priority class only).
4416	*/
4417	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4418	{
4419	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4420	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4421	RT_NOREF2(pVCpu, pVmcsInfo);
4422	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4423	}
4424
4425
4426	/**
4427	* Exports the guest APIC TPR state into the VMCS.
4428	*
4429	* @returns VBox status code.
4430	* @param pVCpu The cross context virtual CPU structure.
4431	* @param pVmxTransient The VMX-transient structure.
4432	*
4433	* @remarks No-long-jump zone!!!
4434	*/
4435	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4436	{
4437	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4438	{
4439	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4440
4441	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4442	if (!pVmxTransient->fIsNestedGuest)
4443	{
4444	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4445	&& APICIsEnabled(pVCpu))
4446	{
4447	/*
4448	* Setup TPR shadowing.
4449	*/
4450	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4451	{
4452	bool fPendingIntr = false;
4453	uint8_t u8Tpr = 0;
4454	uint8_t u8PendingIntr = 0;
4455	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4456	AssertRCReturn(rc, rc);
4457
4458	/*
4459	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4460	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4461	* priority of the pending interrupt so we can deliver the interrupt. If there
4462	* are no interrupts pending, set threshold to 0 to not cause any
4463	* TPR-below-threshold VM-exits.
4464	*/
4465	Assert(pVmcsInfo->pbVirtApic);
4466	pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
4467	uint32_t u32TprThreshold = 0;
4468	if (fPendingIntr)
4469	{
4470	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4471	(which is the Task-Priority Class). */
4472	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4473	const uint8_t u8TprPriority = u8Tpr >> 4;
4474	if (u8PendingPriority <= u8TprPriority)
4475	u32TprThreshold = u8PendingPriority;
4476	}
4477
4478	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
4479	AssertRCReturn(rc, rc);
4480	}
4481	}
4482	}
4483	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4484	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4485	}
4486	return VINF_SUCCESS;
4487	}
4488
4489
4490	/**
4491	* Gets the guest interruptibility-state.
4492	*
4493	* @returns Guest's interruptibility-state.
4494	* @param pVCpu The cross context virtual CPU structure.
4495	* @param pVmcsInfo The VMCS info. object.
4496	*
4497	* @remarks No-long-jump zone!!!
4498	*/
4499	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
4500	{
4501	/*
4502	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4503	*/
4504	uint32_t fIntrState = 0;
4505	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4506	{
4507	/* If inhibition is active, RIP and RFLAGS should've been updated
4508	(i.e. read previously from the VMCS or from ring-3). */
4509	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4510	#ifdef VBOX_STRICT
4511	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
4512	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4513	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
4514	#endif
4515	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
4516	{
4517	if (pCtx->eflags.Bits.u1IF)
4518	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4519	else
4520	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
4521	}
4522	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4523	{
4524	/*
4525	* We can clear the inhibit force flag as even if we go back to the recompiler
4526	* without executing guest code in VT-x, the flag's condition to be cleared is
4527	* met and thus the cleared state is correct.
4528	*/
4529	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4530	}
4531	}
4532
4533	/*
4534	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
4535	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
4536	* setting this would block host-NMIs and IRET will not clear the blocking.
4537	*
4538	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
4539	*
4540	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
4541	*/
4542	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
4543	&& CPUMIsGuestNmiBlocking(pVCpu))
4544	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
4545
4546	return fIntrState;
4547	}
4548
4549
4550	/**
4551	* Exports the exception intercepts required for guest execution in the VMCS.
4552	*
4553	* @returns VBox status code.
4554	* @param pVCpu The cross context virtual CPU structure.
4555	* @param pVmxTransient The VMX-transient structure.
4556	*
4557	* @remarks No-long-jump zone!!!
4558	*/
4559	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4560	{
4561	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
4562	{
4563	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
4564	if ( !pVmxTransient->fIsNestedGuest
4565	&& pVCpu->hm.s.fGIMTrapXcptUD)
4566	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
4567	else
4568	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
4569
4570	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
4571	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
4572	}
4573	return VINF_SUCCESS;
4574	}
4575
4576
4577	/**
4578	* Exports the guest's RIP into the guest-state area in the VMCS.
4579	*
4580	* @returns VBox status code.
4581	* @param pVCpu The cross context virtual CPU structure.
4582	*
4583	* @remarks No-long-jump zone!!!
4584	*/
4585	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
4586	{
4587	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4588	{
4589	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
4590
4591	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
4592	AssertRCReturn(rc, rc);
4593
4594	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
4595	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
4596	}
4597	return VINF_SUCCESS;
4598	}
4599
4600
4601	/**
4602	* Exports the guest's RSP into the guest-state area in the VMCS.
4603	*
4604	* @returns VBox status code.
4605	* @param pVCpu The cross context virtual CPU structure.
4606	*
4607	* @remarks No-long-jump zone!!!
4608	*/
4609	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
4610	{
4611	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
4612	{
4613	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
4614
4615	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
4616	AssertRCReturn(rc, rc);
4617
4618	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
4619	}
4620	return VINF_SUCCESS;
4621	}
4622
4623
4624	/**
4625	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
4626	*
4627	* @returns VBox status code.
4628	* @param pVCpu The cross context virtual CPU structure.
4629	* @param pVmxTransient The VMX-transient structure.
4630	*
4631	* @remarks No-long-jump zone!!!
4632	*/
4633	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4634	{
4635	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
4636	{
4637	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
4638
4639	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
4640	Let us assert it as such and use 32-bit VMWRITE. */
4641	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
4642	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
4643	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
4644	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
4645
4646	/*
4647	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
4648	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
4649	* can run the real-mode guest code under Virtual 8086 mode.
4650	*/
4651	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4652	if (pVmcsInfo->RealMode.fRealOnV86Active)
4653	{
4654	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4655	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4656	Assert(!pVmxTransient->fIsNestedGuest);
4657	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
4658	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
4659	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
4660	}
4661
4662	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
4663	AssertRCReturn(rc, rc);
4664
4665	/*
4666	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
4667	*
4668	* We must avoid setting any automatic debug exceptions delivery when single-stepping
4669	* through the hypervisor debugger using EFLAGS.TF.
4670	*/
4671	if ( !pVmxTransient->fIsNestedGuest
4672	&& !pVCpu->hm.s.fSingleInstruction
4673	&& fEFlags.Bits.u1TF)
4674	{
4675	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
4676	* premature trips to ring-3 esp since IEM does not yet handle it. */
4677	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
4678	AssertRCReturn(rc, rc);
4679	}
4680	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
4681	* nested-guest VMCS. */
4682
4683	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
4684	Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
4685	}
4686	return VINF_SUCCESS;
4687	}
4688
4689
4690	/**
4691	* Exports the guest CR0 control register into the guest-state area in the VMCS.
4692	*
4693	* The guest FPU state is always pre-loaded hence we don't need to bother about
4694	* sharing FPU related CR0 bits between the guest and host.
4695	*
4696	* @returns VBox status code.
4697	* @param pVCpu The cross context virtual CPU structure.
4698	* @param pVmxTransient The VMX-transient structure.
4699	*
4700	* @remarks No-long-jump zone!!!
4701	*/
4702	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4703	{
4704	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
4705	{
4706	PVM pVM = pVCpu->CTX_SUFF(pVM);
4707	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4708
4709	/*
4710	* Figure out fixed CR0 bits in VMX operation.
4711	*/
4712	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4713	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4714	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4715	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
4716	else
4717	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
4718
4719	if (!pVmxTransient->fIsNestedGuest)
4720	{
4721	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4722	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4723	uint64_t const u64ShadowCr0 = u64GuestCr0;
4724	Assert(!RT_HI_U32(u64GuestCr0));
4725
4726	/*
4727	* Setup VT-x's view of the guest CR0.
4728	*/
4729	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
4730	if (pVM->hm.s.fNestedPaging)
4731	{
4732	if (CPUMIsGuestPagingEnabled(pVCpu))
4733	{
4734	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
4735	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
4736	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
4737	}
4738	else
4739	{
4740	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
4741	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
4742	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
4743	}
4744
4745	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
4746	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4747	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
4748	}
4749	else
4750	{
4751	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
4752	u64GuestCr0 \|= X86_CR0_WP;
4753	}
4754
4755	/*
4756	* Guest FPU bits.
4757	*
4758	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
4759	* using CR0.TS.
4760	*
4761	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
4762	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
4763	*/
4764	u64GuestCr0 \|= X86_CR0_NE;
4765
4766	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
4767	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
4768
4769	/*
4770	* Update exception intercepts.
4771	*/
4772	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
4773	if (pVmcsInfo->RealMode.fRealOnV86Active)
4774	{
4775	Assert(PDMVmmDevHeapIsEnabled(pVM));
4776	Assert(pVM->hm.s.vmx.pRealModeTSS);
4777	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
4778	}
4779	else
4780	{
4781	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
4782	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
4783	if (fInterceptMF)
4784	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
4785	}
4786
4787	/* Additional intercepts for debugging, define these yourself explicitly. */
4788	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4789	uXcptBitmap \|= 0
4790	\| RT_BIT(X86_XCPT_BP)
4791	\| RT_BIT(X86_XCPT_DE)
4792	\| RT_BIT(X86_XCPT_NM)
4793	\| RT_BIT(X86_XCPT_TS)
4794	\| RT_BIT(X86_XCPT_UD)
4795	\| RT_BIT(X86_XCPT_NP)
4796	\| RT_BIT(X86_XCPT_SS)
4797	\| RT_BIT(X86_XCPT_GP)
4798	\| RT_BIT(X86_XCPT_PF)
4799	\| RT_BIT(X86_XCPT_MF)
4800	;
4801	#elif defined(HMVMX_ALWAYS_TRAP_PF)
4802	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
4803	#endif
4804	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
4805	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
4806	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
4807
4808	/* Apply the fixed CR0 bits and enable caching. */
4809	u64GuestCr0 \|= fSetCr0;
4810	u64GuestCr0 &= fZapCr0;
4811	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4812
4813	/* Commit the CR0 and related fields to the guest VMCS. */
4814	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo Fix to 64-bit when we drop 32-bit. */
4815	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4816	if (uProcCtls != pVmcsInfo->u32ProcCtls)
4817	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4818	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
4819	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
4820	AssertRCReturn(rc, rc);
4821
4822	/* Update our caches. */
4823	pVmcsInfo->u32ProcCtls = uProcCtls;
4824	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
4825
4826	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4827	}
4828	else
4829	{
4830	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
4831	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4832	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4833	uint64_t const u64ShadowCr0 = pVmcsNstGst->u64Cr0ReadShadow.u;
4834	Assert(!RT_HI_U32(u64GuestCr0));
4835	Assert(u64GuestCr0 & X86_CR0_NE);
4836
4837	/* Apply the fixed CR0 bits and enable caching. */
4838	u64GuestCr0 \|= fSetCr0;
4839	u64GuestCr0 &= fZapCr0;
4840	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4841
4842	/* Commit the CR0 and CR0 read shadow to the nested-guest VMCS. */
4843	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo NSTVMX: Fix to 64-bit when we drop 32-bit. */
4844	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4845	AssertRCReturn(rc, rc);
4846
4847	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4848	}
4849
4850	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
4851	}
4852
4853	return VINF_SUCCESS;
4854	}
4855
4856
4857	/**
4858	* Exports the guest control registers (CR3, CR4) into the guest-state area
4859	* in the VMCS.
4860	*
4861	* @returns VBox strict status code.
4862	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
4863	* without unrestricted guest access and the VMMDev is not presently
4864	* mapped (e.g. EFI32).
4865	*
4866	* @param pVCpu The cross context virtual CPU structure.
4867	* @param pVmxTransient The VMX-transient structure.
4868	*
4869	* @remarks No-long-jump zone!!!
4870	*/
4871	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4872	{
4873	int rc = VINF_SUCCESS;
4874	PVM pVM = pVCpu->CTX_SUFF(pVM);
4875
4876	/*
4877	* Guest CR2.
4878	* It's always loaded in the assembler code. Nothing to do here.
4879	*/
4880
4881	/*
4882	* Guest CR3.
4883	*/
4884	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
4885	{
4886	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
4887
4888	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
4889	if (pVM->hm.s.fNestedPaging)
4890	{
4891	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4892	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
4893
4894	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
4895	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
4896	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
4897	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
4898
4899	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
4900	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
4901	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
4902
4903	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
4904	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
4905	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
4906	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
4907	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
4908	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
4909	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
4910
4911	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
4912	AssertRCReturn(rc, rc);
4913
4914	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4915	if ( pVM->hm.s.vmx.fUnrestrictedGuest
4916	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
4917	{
4918	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
4919	if (CPUMIsGuestInPAEModeEx(pCtx))
4920	{
4921	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
4922	AssertRCReturn(rc, rc);
4923	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
4924	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
4925	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
4926	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
4927	AssertRCReturn(rc, rc);
4928	}
4929
4930	/*
4931	* The guest's view of its CR3 is unblemished with nested paging when the
4932	* guest is using paging or we have unrestricted guest execution to handle
4933	* the guest when it's not using paging.
4934	*/
4935	GCPhysGuestCR3 = pCtx->cr3;
4936	}
4937	else
4938	{
4939	/*
4940	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
4941	* thinks it accesses physical memory directly, we use our identity-mapped
4942	* page table to map guest-linear to guest-physical addresses. EPT takes care
4943	* of translating it to host-physical addresses.
4944	*/
4945	RTGCPHYS GCPhys;
4946	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4947
4948	/* We obtain it here every time as the guest could have relocated this PCI region. */
4949	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4950	if (RT_SUCCESS(rc))
4951	{ /* likely */ }
4952	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4953	{
4954	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
4955	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4956	}
4957	else
4958	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4959
4960	GCPhysGuestCR3 = GCPhys;
4961	}
4962
4963	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
4964	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4965	AssertRCReturn(rc, rc);
4966	}
4967	else
4968	{
4969	/* Non-nested paging case, just use the hypervisor's CR3. */
4970	RTHCPHYS const HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4971
4972	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
4973	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4974	AssertRCReturn(rc, rc);
4975	}
4976
4977	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
4978	}
4979
4980	/*
4981	* Guest CR4.
4982	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4983	*/
4984	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
4985	{
4986	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4987	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4988	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
4989
4990	/*
4991	* Figure out fixed CR4 bits in VMX operation.
4992	*/
4993	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
4994	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
4995
4996	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
4997	uint64_t u64GuestCr4 = pCtx->cr4;
4998	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest ? pCtx->cr4 : pVmcsNstGst->u64Cr4ReadShadow.u;
4999	Assert(!RT_HI_U32(u64GuestCr4));
5000
5001	/*
5002	* Setup VT-x's view of the guest CR4.
5003	*
5004	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5005	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5006	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5007	*
5008	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5009	*/
5010	if (pVmcsInfo->RealMode.fRealOnV86Active)
5011	{
5012	Assert(pVM->hm.s.vmx.pRealModeTSS);
5013	Assert(PDMVmmDevHeapIsEnabled(pVM));
5014	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5015	}
5016
5017	if (pVM->hm.s.fNestedPaging)
5018	{
5019	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5020	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5021	{
5022	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5023	u64GuestCr4 \|= X86_CR4_PSE;
5024	/* Our identity mapping is a 32-bit page directory. */
5025	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5026	}
5027	/* else use guest CR4.*/
5028	}
5029	else
5030	{
5031	Assert(!pVmxTransient->fIsNestedGuest);
5032
5033	/*
5034	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5035	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5036	*/
5037	switch (pVCpu->hm.s.enmShadowMode)
5038	{
5039	case PGMMODE_REAL: /* Real-mode. */
5040	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5041	case PGMMODE_32_BIT: /* 32-bit paging. */
5042	{
5043	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5044	break;
5045	}
5046
5047	case PGMMODE_PAE: /* PAE paging. */
5048	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5049	{
5050	u64GuestCr4 \|= X86_CR4_PAE;
5051	break;
5052	}
5053
5054	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5055	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5056	#ifdef VBOX_ENABLE_64_BITS_GUESTS
5057	break;
5058	#endif
5059	default:
5060	AssertFailed();
5061	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5062	}
5063	}
5064
5065	/* Apply the fixed CR4 bits (mainly CR4.VMXE). */
5066	u64GuestCr4 \|= fSetCr4;
5067	u64GuestCr4 &= fZapCr4;
5068
5069	/* Commit the CR4 and CR4 read shadow to the guest VMCS. */
5070	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u64GuestCr4); /** @todo Fix to 64-bit when we drop 32-bit. */
5071	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
5072	AssertRCReturn(rc, rc);
5073
5074	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5075	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5076
5077	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5078
5079	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5080	}
5081	return rc;
5082	}
5083
5084
5085	/**
5086	* Exports the guest debug registers into the guest-state area in the VMCS.
5087	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5088	*
5089	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5090	*
5091	* @returns VBox status code.
5092	* @param pVCpu The cross context virtual CPU structure.
5093	* @param pVmxTransient The VMX-transient structure.
5094	*
5095	* @remarks No-long-jump zone!!!
5096	*/
5097	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5098	{
5099	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5100
5101	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5102	* stepping. */
5103	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5104	if (pVmxTransient->fIsNestedGuest)
5105	{
5106	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5107	AssertRCReturn(rc, rc);
5108	return VINF_SUCCESS;
5109	}
5110
5111	#ifdef VBOX_STRICT
5112	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5113	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5114	{
5115	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5116	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5117	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5118	}
5119	#endif
5120
5121	bool fSteppingDB = false;
5122	bool fInterceptMovDRx = false;
5123	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5124	if (pVCpu->hm.s.fSingleInstruction)
5125	{
5126	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5127	PVM pVM = pVCpu->CTX_SUFF(pVM);
5128	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5129	{
5130	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5131	Assert(fSteppingDB == false);
5132	}
5133	else
5134	{
5135	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5136	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5137	pVCpu->hm.s.fClearTrapFlag = true;
5138	fSteppingDB = true;
5139	}
5140	}
5141
5142	uint32_t u32GuestDr7;
5143	if ( fSteppingDB
5144	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5145	{
5146	/*
5147	* Use the combined guest and host DRx values found in the hypervisor register set
5148	* because the hypervisor debugger has breakpoints active or someone is single stepping
5149	* on the host side without a monitor trap flag.
5150	*
5151	* Note! DBGF expects a clean DR6 state before executing guest code.
5152	*/
5153	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5154	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5155	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
5156	{
5157	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5158	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
5159	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
5160	}
5161	else
5162	#endif
5163	if (!CPUMIsHyperDebugStateActive(pVCpu))
5164	{
5165	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5166	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5167	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5168	}
5169
5170	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5171	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
5172	pVCpu->hm.s.fUsingHyperDR7 = true;
5173	fInterceptMovDRx = true;
5174	}
5175	else
5176	{
5177	/*
5178	* If the guest has enabled debug registers, we need to load them prior to
5179	* executing guest code so they'll trigger at the right time.
5180	*/
5181	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5182	{
5183	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5184	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5185	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
5186	{
5187	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5188	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
5189	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
5190	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5191	}
5192	else
5193	#endif
5194	if (!CPUMIsGuestDebugStateActive(pVCpu))
5195	{
5196	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5197	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5198	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5199	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5200	}
5201	Assert(!fInterceptMovDRx);
5202	}
5203	/*
5204	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5205	* must intercept #DB in order to maintain a correct DR6 guest value, and
5206	* because we need to intercept it to prevent nested #DBs from hanging the
5207	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5208	*/
5209	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5210	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
5211	&& !CPUMIsGuestDebugStateActive(pVCpu))
5212	#else
5213	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5214	#endif
5215	{
5216	fInterceptMovDRx = true;
5217	}
5218
5219	/* Update DR7 with the actual guest value. */
5220	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5221	pVCpu->hm.s.fUsingHyperDR7 = false;
5222	}
5223
5224	if (fInterceptMovDRx)
5225	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5226	else
5227	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5228
5229	/*
5230	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5231	* monitor-trap flag and update our cache.
5232	*/
5233	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5234	{
5235	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5236	AssertRCReturn(rc2, rc2);
5237	pVmcsInfo->u32ProcCtls = uProcCtls;
5238	}
5239
5240	/*
5241	* Update guest DR7.
5242	*/
5243	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
5244	AssertRCReturn(rc, rc);
5245
5246	/*
5247	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
5248	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
5249	*
5250	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5251	*/
5252	if (fSteppingDB)
5253	{
5254	Assert(pVCpu->hm.s.fSingleInstruction);
5255	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
5256
5257	uint32_t fIntrState = 0;
5258	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
5259	AssertRCReturn(rc, rc);
5260
5261	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5262	{
5263	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5264	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5265	AssertRCReturn(rc, rc);
5266	}
5267	}
5268
5269	return VINF_SUCCESS;
5270	}
5271
5272
5273	#ifdef VBOX_STRICT
5274	/**
5275	* Strict function to validate segment registers.
5276	*
5277	* @param pVCpu The cross context virtual CPU structure.
5278	* @param pVmcsInfo The VMCS info. object.
5279	*
5280	* @remarks Will import guest CR0 on strict builds during validation of
5281	* segments.
5282	*/
5283	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
5284	{
5285	/*
5286	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5287	*
5288	* The reason we check for attribute value 0 in this function and not just the unusable bit is
5289	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
5290	* unusable bit and doesn't change the guest-context value.
5291	*/
5292	PVM pVM = pVCpu->CTX_SUFF(pVM);
5293	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5294	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
5295	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
5296	&& ( !CPUMIsGuestInRealModeEx(pCtx)
5297	&& !CPUMIsGuestInV86ModeEx(pCtx)))
5298	{
5299	/* Protected mode checks */
5300	/* CS */
5301	Assert(pCtx->cs.Attr.n.u1Present);
5302	Assert(!(pCtx->cs.Attr.u & 0xf00));
5303	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
5304	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5305	\|\| !(pCtx->cs.Attr.n.u1Granularity));
5306	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
5307	\|\| (pCtx->cs.Attr.n.u1Granularity));
5308	/* CS cannot be loaded with NULL in protected mode. */
5309	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
5310	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5311	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
5312	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5313	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
5314	else
5315	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
5316	/* SS */
5317	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5318	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
5319	if ( !(pCtx->cr0 & X86_CR0_PE)
5320	\|\| pCtx->cs.Attr.n.u4Type == 3)
5321	{
5322	Assert(!pCtx->ss.Attr.n.u2Dpl);
5323	}
5324	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
5325	{
5326	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5327	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
5328	Assert(pCtx->ss.Attr.n.u1Present);
5329	Assert(!(pCtx->ss.Attr.u & 0xf00));
5330	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
5331	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
5332	\|\| !(pCtx->ss.Attr.n.u1Granularity));
5333	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
5334	\|\| (pCtx->ss.Attr.n.u1Granularity));
5335	}
5336	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
5337	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
5338	{
5339	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5340	Assert(pCtx->ds.Attr.n.u1Present);
5341	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
5342	Assert(!(pCtx->ds.Attr.u & 0xf00));
5343	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
5344	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
5345	\|\| !(pCtx->ds.Attr.n.u1Granularity));
5346	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
5347	\|\| (pCtx->ds.Attr.n.u1Granularity));
5348	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5349	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
5350	}
5351	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
5352	{
5353	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5354	Assert(pCtx->es.Attr.n.u1Present);
5355	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
5356	Assert(!(pCtx->es.Attr.u & 0xf00));
5357	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
5358	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
5359	\|\| !(pCtx->es.Attr.n.u1Granularity));
5360	Assert( !(pCtx->es.u32Limit & 0xfff00000)
5361	\|\| (pCtx->es.Attr.n.u1Granularity));
5362	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5363	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
5364	}
5365	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
5366	{
5367	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5368	Assert(pCtx->fs.Attr.n.u1Present);
5369	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
5370	Assert(!(pCtx->fs.Attr.u & 0xf00));
5371	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
5372	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
5373	\|\| !(pCtx->fs.Attr.n.u1Granularity));
5374	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
5375	\|\| (pCtx->fs.Attr.n.u1Granularity));
5376	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5377	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5378	}
5379	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
5380	{
5381	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5382	Assert(pCtx->gs.Attr.n.u1Present);
5383	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
5384	Assert(!(pCtx->gs.Attr.u & 0xf00));
5385	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
5386	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
5387	\|\| !(pCtx->gs.Attr.n.u1Granularity));
5388	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
5389	\|\| (pCtx->gs.Attr.n.u1Granularity));
5390	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5391	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5392	}
5393	/* 64-bit capable CPUs. */
5394	# if HC_ARCH_BITS == 64
5395	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5396	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
5397	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
5398	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
5399	# endif
5400	}
5401	else if ( CPUMIsGuestInV86ModeEx(pCtx)
5402	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
5403	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
5404	{
5405	/* Real and v86 mode checks. */
5406	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
5407	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
5408	if (pVmcsInfo->RealMode.fRealOnV86Active)
5409	{
5410	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
5411	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
5412	}
5413	else
5414	{
5415	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
5416	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
5417	}
5418
5419	/* CS */
5420	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
5421	Assert(pCtx->cs.u32Limit == 0xffff);
5422	Assert(u32CSAttr == 0xf3);
5423	/* SS */
5424	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
5425	Assert(pCtx->ss.u32Limit == 0xffff);
5426	Assert(u32SSAttr == 0xf3);
5427	/* DS */
5428	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
5429	Assert(pCtx->ds.u32Limit == 0xffff);
5430	Assert(u32DSAttr == 0xf3);
5431	/* ES */
5432	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
5433	Assert(pCtx->es.u32Limit == 0xffff);
5434	Assert(u32ESAttr == 0xf3);
5435	/* FS */
5436	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
5437	Assert(pCtx->fs.u32Limit == 0xffff);
5438	Assert(u32FSAttr == 0xf3);
5439	/* GS */
5440	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
5441	Assert(pCtx->gs.u32Limit == 0xffff);
5442	Assert(u32GSAttr == 0xf3);
5443	/* 64-bit capable CPUs. */
5444	# if HC_ARCH_BITS == 64
5445	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5446	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
5447	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
5448	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
5449	# endif
5450	}
5451	}
5452	#endif /* VBOX_STRICT */
5453
5454
5455	/**
5456	* Exports a guest segment register into the guest-state area in the VMCS.
5457	*
5458	* @returns VBox status code.
5459	* @param pVCpu The cross context virtual CPU structure.
5460	* @param pVmcsInfo The VMCS info. object.
5461	* @param iSegReg The segment register number (X86_SREG_XXX).
5462	* @param pSelReg Pointer to the segment selector.
5463	*
5464	* @remarks No-long-jump zone!!!
5465	*/
5466	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
5467	{
5468	Assert(iSegReg < X86_SREG_COUNT);
5469	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
5470	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
5471	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
5472	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
5473
5474	uint32_t u32Access = pSelReg->Attr.u;
5475	if (pVmcsInfo->RealMode.fRealOnV86Active)
5476	{
5477	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
5478	u32Access = 0xf3;
5479	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5480	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5481	RT_NOREF_PV(pVCpu);
5482	}
5483	else
5484	{
5485	/*
5486	* The way to differentiate between whether this is really a null selector or was just
5487	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
5488	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
5489	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
5490	* NULL selectors loaded in protected-mode have their attribute as 0.
5491	*/
5492	if (!u32Access)
5493	u32Access = X86DESCATTR_UNUSABLE;
5494	}
5495
5496	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
5497	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
5498	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
5499
5500	/*
5501	* Commit it to the VMCS.
5502	*/
5503	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
5504	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
5505	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
5506	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
5507	AssertRCReturn(rc, rc);
5508	return rc;
5509	}
5510
5511
5512	/**
5513	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
5514	* area in the VMCS.
5515	*
5516	* @returns VBox status code.
5517	* @param pVCpu The cross context virtual CPU structure.
5518	* @param pVmxTransient The VMX-transient structure.
5519	*
5520	* @remarks Will import guest CR0 on strict builds during validation of
5521	* segments.
5522	* @remarks No-long-jump zone!!!
5523	*/
5524	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5525	{
5526	int rc = VERR_INTERNAL_ERROR_5;
5527	PVM pVM = pVCpu->CTX_SUFF(pVM);
5528	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5529	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5530
5531	/*
5532	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
5533	*/
5534	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
5535	{
5536	#ifdef VBOX_WITH_REM
5537	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
5538	{
5539	Assert(!pVmxTransient->fIsNestedGuest);
5540	Assert(pVM->hm.s.vmx.pRealModeTSS);
5541	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
5542	if ( pVmcsInfo->fWasInRealMode
5543	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
5544	{
5545	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
5546	in real-mode (e.g. OpenBSD 4.0) */
5547	REMFlushTBs(pVM);
5548	Log4Func(("Switch to protected mode detected!\n"));
5549	pVmcsInfo->fWasInRealMode = false;
5550	}
5551	}
5552	#endif
5553	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
5554	{
5555	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
5556	if (pVmcsInfo->RealMode.fRealOnV86Active)
5557	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
5558	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
5559	AssertRCReturn(rc, rc);
5560	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
5561	}
5562
5563	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
5564	{
5565	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
5566	if (pVmcsInfo->RealMode.fRealOnV86Active)
5567	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
5568	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
5569	AssertRCReturn(rc, rc);
5570	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
5571	}
5572
5573	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
5574	{
5575	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
5576	if (pVmcsInfo->RealMode.fRealOnV86Active)
5577	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
5578	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
5579	AssertRCReturn(rc, rc);
5580	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
5581	}
5582
5583	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
5584	{
5585	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
5586	if (pVmcsInfo->RealMode.fRealOnV86Active)
5587	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
5588	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
5589	AssertRCReturn(rc, rc);
5590	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
5591	}
5592
5593	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
5594	{
5595	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
5596	if (pVmcsInfo->RealMode.fRealOnV86Active)
5597	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
5598	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
5599	AssertRCReturn(rc, rc);
5600	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
5601	}
5602
5603	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
5604	{
5605	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
5606	if (pVmcsInfo->RealMode.fRealOnV86Active)
5607	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
5608	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
5609	AssertRCReturn(rc, rc);
5610	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
5611	}
5612
5613	#ifdef VBOX_STRICT
5614	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
5615	#endif
5616	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
5617	pCtx->cs.Attr.u));
5618	}
5619
5620	/*
5621	* Guest TR.
5622	*/
5623	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
5624	{
5625	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
5626
5627	/*
5628	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
5629	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
5630	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
5631	*/
5632	uint16_t u16Sel;
5633	uint32_t u32Limit;
5634	uint64_t u64Base;
5635	uint32_t u32AccessRights;
5636	if (!pVmcsInfo->RealMode.fRealOnV86Active)
5637	{
5638	u16Sel = pCtx->tr.Sel;
5639	u32Limit = pCtx->tr.u32Limit;
5640	u64Base = pCtx->tr.u64Base;
5641	u32AccessRights = pCtx->tr.Attr.u;
5642	}
5643	else
5644	{
5645	Assert(!pVmxTransient->fIsNestedGuest);
5646	Assert(pVM->hm.s.vmx.pRealModeTSS);
5647	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
5648
5649	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
5650	RTGCPHYS GCPhys;
5651	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
5652	AssertRCReturn(rc, rc);
5653
5654	X86DESCATTR DescAttr;
5655	DescAttr.u = 0;
5656	DescAttr.n.u1Present = 1;
5657	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
5658
5659	u16Sel = 0;
5660	u32Limit = HM_VTX_TSS_SIZE;
5661	u64Base = GCPhys;
5662	u32AccessRights = DescAttr.u;
5663	}
5664
5665	/* Validate. */
5666	Assert(!(u16Sel & RT_BIT(2)));
5667	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
5668	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
5669	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
5670	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
5671	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
5672	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
5673	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
5674	Assert( (u32Limit & 0xfff) == 0xfff
5675	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
5676	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
5677	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
5678
5679	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
5680	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
5681	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
5682	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
5683	AssertRCReturn(rc, rc);
5684
5685	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
5686	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
5687	}
5688
5689	/*
5690	* Guest GDTR.
5691	*/
5692	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
5693	{
5694	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
5695
5696	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
5697	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
5698	AssertRCReturn(rc, rc);
5699
5700	/* Validate. */
5701	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5702
5703	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
5704	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
5705	}
5706
5707	/*
5708	* Guest LDTR.
5709	*/
5710	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
5711	{
5712	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
5713
5714	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
5715	uint32_t u32Access;
5716	if ( !pVmxTransient->fIsNestedGuest
5717	&& !pCtx->ldtr.Attr.u)
5718	u32Access = X86DESCATTR_UNUSABLE;
5719	else
5720	u32Access = pCtx->ldtr.Attr.u;
5721
5722	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
5723	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
5724	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
5725	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
5726	AssertRCReturn(rc, rc);
5727
5728	/* Validate. */
5729	if (!(u32Access & X86DESCATTR_UNUSABLE))
5730	{
5731	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
5732	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
5733	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
5734	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
5735	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
5736	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
5737	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
5738	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
5739	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
5740	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
5741	}
5742
5743	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
5744	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
5745	}
5746
5747	/*
5748	* Guest IDTR.
5749	*/
5750	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
5751	{
5752	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
5753
5754	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
5755	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
5756	AssertRCReturn(rc, rc);
5757
5758	/* Validate. */
5759	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5760
5761	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
5762	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
5763	}
5764
5765	return VINF_SUCCESS;
5766	}
5767
5768
5769	/**
5770	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
5771	* areas.
5772	*
5773	* These MSRs will automatically be loaded to the host CPU on every successful
5774	* VM-entry and stored from the host CPU on every successful VM-exit.
5775	*
5776	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
5777	* actual host MSR values are not- updated here for performance reasons. See
5778	* hmR0VmxExportHostMsrs().
5779	*
5780	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
5781	*
5782	* @returns VBox status code.
5783	* @param pVCpu The cross context virtual CPU structure.
5784	* @param pVmxTransient The VMX-transient structure.
5785	*
5786	* @remarks No-long-jump zone!!!
5787	*/
5788	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5789	{
5790	AssertPtr(pVCpu);
5791	AssertPtr(pVmxTransient);
5792
5793	PVM pVM = pVCpu->CTX_SUFF(pVM);
5794	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5795
5796	/*
5797	* MSRs that we use the auto-load/store MSR area in the VMCS.
5798	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
5799	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
5800	*
5801	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
5802	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
5803	* emulation, nothing to do here.
5804	*/
5805	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
5806	{
5807	if ( !pVmxTransient->fIsNestedGuest
5808	&& pVM->hm.s.fAllow64BitGuests)
5809	{
5810	#if HC_ARCH_BITS == 32
5811	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
5812	Assert(!pVmxTransient->fIsNestedGuest);
5813
5814	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
5815	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
5816	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
5817	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
5818	AssertRCReturn(rc, rc);
5819	#endif
5820	}
5821	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
5822	}
5823
5824	/*
5825	* Guest Sysenter MSRs.
5826	*/
5827	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
5828	{
5829	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
5830
5831	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
5832	{
5833	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
5834	AssertRCReturn(rc, rc);
5835	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
5836	}
5837
5838	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
5839	{
5840	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
5841	AssertRCReturn(rc, rc);
5842	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
5843	}
5844
5845	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
5846	{
5847	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
5848	AssertRCReturn(rc, rc);
5849	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
5850	}
5851	}
5852
5853	/*
5854	* Guest/host EFER MSR.
5855	*/
5856	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
5857	{
5858	/* Whether we are using the VMCS to swap the EFER MSR must have been
5859	determined earlier while exporting VM-entry/VM-exit controls. */
5860	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
5861	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
5862
5863	if (hmR0VmxShouldSwapEferMsr(pVCpu))
5864	{
5865	/*
5866	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
5867	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
5868	*/
5869	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
5870	{
5871	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
5872	AssertRCReturn(rc, rc);
5873	}
5874	else
5875	{
5876	/*
5877	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
5878	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
5879	*/
5880	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
5881	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5882	AssertRCReturn(rc, rc);
5883	}
5884	}
5885	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
5886	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
5887
5888	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
5889	}
5890
5891	/*
5892	* Other MSRs.
5893	* Speculation Control (R/W).
5894	*/
5895	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
5896	{
5897	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
5898	if (pVM->cpum.ro.GuestFeatures.fIbrs)
5899	{
5900	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
5901	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5902	AssertRCReturn(rc, rc);
5903	}
5904	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
5905	}
5906
5907	return VINF_SUCCESS;
5908	}
5909
5910
5911	/**
5912	* Selects up the appropriate function to run guest code.
5913	*
5914	* @returns VBox status code.
5915	* @param pVCpu The cross context virtual CPU structure.
5916	* @param pVmxTransient The VMX-transient structure.
5917	*
5918	* @remarks No-long-jump zone!!!
5919	*/
5920	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5921	{
5922	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5923	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5924
5925	if (CPUMIsGuestInLongModeEx(pCtx))
5926	{
5927	#ifndef VBOX_ENABLE_64_BITS_GUESTS
5928	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5929	#endif
5930	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
5931	#if HC_ARCH_BITS == 32
5932	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
5933	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
5934	{
5935	#ifdef VBOX_STRICT
5936	if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5937	{
5938	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5939	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5940	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5941	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5942	("fCtxChanged=%#RX64\n", fCtxChanged));
5943	}
5944	#endif
5945	pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
5946
5947	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
5948	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
5949	pVmcsInfo->fSwitchedTo64on32 = true;
5950	Log4Func(("Selected 64-bit switcher\n"));
5951	}
5952	#else
5953	/* 64-bit host. */
5954	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
5955	#endif
5956	}
5957	else
5958	{
5959	/* Guest is not in long mode, use the 32-bit handler. */
5960	#if HC_ARCH_BITS == 32
5961	if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
5962	&& !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
5963	&& pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5964	{
5965	# ifdef VBOX_STRICT
5966	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5967	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5968	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5969	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5970	("fCtxChanged=%#RX64\n", fCtxChanged));
5971	# endif
5972	}
5973	# ifdef VBOX_ENABLE_64_BITS_GUESTS
5974	/*
5975	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
5976	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
5977	* switcher flag now because we know the guest is in a sane state where it's safe
5978	* to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
5979	* the much faster 32-bit switcher again.
5980	*/
5981	if (!pVmcsInfo->fSwitchedTo64on32)
5982	{
5983	if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
5984	Log4Func(("Selected 32-bit switcher\n"));
5985	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5986	}
5987	else
5988	{
5989	Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
5990	if ( pVmcsInfo->RealMode.fRealOnV86Active
5991	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
5992	{
5993	pVmcsInfo->fSwitchedTo64on32 = false;
5994	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5995	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
5996	\| HM_CHANGED_VMX_ENTRY_EXIT_CTLS
5997	\| HM_CHANGED_HOST_CONTEXT);
5998	Log4Func(("Selected 32-bit switcher (safe)\n"));
5999	}
6000	}
6001	# else
6002	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6003	# endif
6004	#else
6005	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6006	#endif
6007	}
6008	Assert(pVmcsInfo->pfnStartVM);
6009	return VINF_SUCCESS;
6010	}
6011
6012
6013	/**
6014	* Wrapper for running the guest code in VT-x.
6015	*
6016	* @returns VBox status code, no informational status codes.
6017	* @param pVCpu The cross context virtual CPU structure.
6018	* @param pVmxTransient The VMX-transient structure.
6019	*
6020	* @remarks No-long-jump zone!!!
6021	*/
6022	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6023	{
6024	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6025	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6026	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6027
6028	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6029
6030	/*
6031	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6032	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6033	* callee-saved and thus the need for this XMM wrapper.
6034	*
6035	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6036	*/
6037	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6038	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6039	PVM pVM = pVCpu->CTX_SUFF(pVM);
6040	#ifdef VBOX_WITH_KERNEL_USING_XMM
6041	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6042	#else
6043	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
6044	#endif
6045	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6046	return rc;
6047	}
6048
6049
6050	/**
6051	* Reports world-switch error and dumps some useful debug info.
6052	*
6053	* @param pVCpu The cross context virtual CPU structure.
6054	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6055	* @param pVmxTransient The VMX-transient structure (only
6056	* exitReason updated).
6057	*/
6058	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6059	{
6060	Assert(pVCpu);
6061	Assert(pVmxTransient);
6062	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6063
6064	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6065	switch (rcVMRun)
6066	{
6067	case VERR_VMX_INVALID_VMXON_PTR:
6068	AssertFailed();
6069	break;
6070	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6071	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6072	{
6073	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6074	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6075	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
6076	AssertRC(rc);
6077
6078	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6079	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6080	Cannot do it here as we may have been long preempted. */
6081
6082	#ifdef VBOX_STRICT
6083	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6084	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6085	pVmxTransient->uExitReason));
6086	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6087	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6088	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6089	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6090	else
6091	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6092	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6093	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6094
6095	/* VMX control bits. */
6096	uint32_t u32Val;
6097	uint64_t u64Val;
6098	RTHCUINTREG uHCReg;
6099	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
6100	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
6101	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
6102	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
6103	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
6104	{
6105	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
6106	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
6107	}
6108	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
6109	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
6110	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
6111	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
6112	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
6113	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
6114	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
6115	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
6116	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
6117	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
6118	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
6119	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
6120	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
6121	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
6122	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
6123	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
6124	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6125	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
6126	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6127	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
6128	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
6129	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
6130	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
6131	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
6132	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
6133	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
6134	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
6135	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
6136	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
6137	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6138	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
6139	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
6140	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
6141	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6142	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6143	{
6144	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
6145	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
6146	}
6147
6148	/* Guest bits. */
6149	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
6150	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
6151	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
6152	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
6153	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
6154	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
6155	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
6156	{
6157	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
6158	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
6159	}
6160
6161	/* Host bits. */
6162	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
6163	Log4(("Host CR0 %#RHr\n", uHCReg));
6164	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
6165	Log4(("Host CR3 %#RHr\n", uHCReg));
6166	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
6167	Log4(("Host CR4 %#RHr\n", uHCReg));
6168
6169	RTGDTR HostGdtr;
6170	PCX86DESCHC pDesc;
6171	ASMGetGDTR(&HostGdtr);
6172	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
6173	Log4(("Host CS %#08x\n", u32Val));
6174	if (u32Val < HostGdtr.cbGdt)
6175	{
6176	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6177	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
6178	}
6179
6180	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
6181	Log4(("Host DS %#08x\n", u32Val));
6182	if (u32Val < HostGdtr.cbGdt)
6183	{
6184	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6185	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
6186	}
6187
6188	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
6189	Log4(("Host ES %#08x\n", u32Val));
6190	if (u32Val < HostGdtr.cbGdt)
6191	{
6192	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6193	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
6194	}
6195
6196	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
6197	Log4(("Host FS %#08x\n", u32Val));
6198	if (u32Val < HostGdtr.cbGdt)
6199	{
6200	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6201	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
6202	}
6203
6204	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
6205	Log4(("Host GS %#08x\n", u32Val));
6206	if (u32Val < HostGdtr.cbGdt)
6207	{
6208	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6209	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
6210	}
6211
6212	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
6213	Log4(("Host SS %#08x\n", u32Val));
6214	if (u32Val < HostGdtr.cbGdt)
6215	{
6216	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6217	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
6218	}
6219
6220	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
6221	Log4(("Host TR %#08x\n", u32Val));
6222	if (u32Val < HostGdtr.cbGdt)
6223	{
6224	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6225	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
6226	}
6227
6228	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
6229	Log4(("Host TR Base %#RHv\n", uHCReg));
6230	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
6231	Log4(("Host GDTR Base %#RHv\n", uHCReg));
6232	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
6233	Log4(("Host IDTR Base %#RHv\n", uHCReg));
6234	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
6235	Log4(("Host SYSENTER CS %#08x\n", u32Val));
6236	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
6237	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
6238	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
6239	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
6240	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
6241	Log4(("Host RSP %#RHv\n", uHCReg));
6242	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
6243	Log4(("Host RIP %#RHv\n", uHCReg));
6244	# if HC_ARCH_BITS == 64
6245	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6246	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6247	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6248	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6249	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6250	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6251	# endif
6252	#endif /* VBOX_STRICT */
6253	break;
6254	}
6255
6256	default:
6257	/* Impossible */
6258	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6259	break;
6260	}
6261	}
6262
6263
6264	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
6265	# ifndef VMX_USE_CACHED_VMCS_ACCESSES
6266	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
6267	# endif
6268
6269	/**
6270	* Initialize the VMCS-Read cache.
6271	*
6272	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
6273	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
6274	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
6275	* (those that have a 32-bit FULL & HIGH part).
6276	*
6277	* @param pVCpu The cross context virtual CPU structure.
6278	*/
6279	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
6280	{
6281	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
6282	do { \
6283	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
6284	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
6285	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
6286	++cReadFields; \
6287	} while (0)
6288
6289	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6290	uint32_t cReadFields = 0;
6291
6292	/*
6293	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
6294	* and serve to indicate exceptions to the rules.
6295	*/
6296
6297	/* Guest-natural selector base fields. */
6298	#if 0
6299	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
6300	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
6301	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
6302	#endif
6303	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
6304	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
6305	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
6306	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
6307	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
6308	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
6309	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
6310	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
6311	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
6312	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
6313	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
6314	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
6315	#if 0
6316	/* Unused natural width guest-state fields. */
6317	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
6318	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
6319	#endif
6320	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
6321	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
6322
6323	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
6324	these 64-bit fields (using "FULL" and "HIGH" fields). */
6325	#if 0
6326	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
6327	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
6328	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
6329	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
6330	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
6331	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
6332	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
6333	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
6334	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
6335	#endif
6336
6337	/* Natural width guest-state fields. */
6338	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
6339	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
6340
6341	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6342	{
6343	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
6344	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
6345	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
6346	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
6347	}
6348	else
6349	{
6350	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
6351	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
6352	}
6353
6354	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
6355	}
6356
6357
6358	/**
6359	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
6360	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
6361	* darwin, running 64-bit guests).
6362	*
6363	* @returns VBox status code.
6364	* @param pVCpu The cross context virtual CPU structure.
6365	* @param idxField The VMCS field encoding.
6366	* @param u64Val 16, 32 or 64-bit value.
6367	*/
6368	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6369	{
6370	int rc;
6371	switch (idxField)
6372	{
6373	/*
6374	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
6375	*/
6376	/* 64-bit Control fields. */
6377	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
6378	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
6379	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
6380	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
6381	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
6382	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
6383	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
6384	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
6385	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
6386	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
6387	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
6388	case VMX_VMCS64_CTRL_EPTP_FULL:
6389	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
6390	/* 64-bit Guest-state fields. */
6391	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
6392	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
6393	case VMX_VMCS64_GUEST_PAT_FULL:
6394	case VMX_VMCS64_GUEST_EFER_FULL:
6395	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
6396	case VMX_VMCS64_GUEST_PDPTE0_FULL:
6397	case VMX_VMCS64_GUEST_PDPTE1_FULL:
6398	case VMX_VMCS64_GUEST_PDPTE2_FULL:
6399	case VMX_VMCS64_GUEST_PDPTE3_FULL:
6400	/* 64-bit Host-state fields. */
6401	case VMX_VMCS64_HOST_PAT_FULL:
6402	case VMX_VMCS64_HOST_EFER_FULL:
6403	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
6404	{
6405	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6406	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
6407	break;
6408	}
6409
6410	/*
6411	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
6412	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
6413	*/
6414	/* Natural-width Guest-state fields. */
6415	case VMX_VMCS_GUEST_CR3:
6416	case VMX_VMCS_GUEST_ES_BASE:
6417	case VMX_VMCS_GUEST_CS_BASE:
6418	case VMX_VMCS_GUEST_SS_BASE:
6419	case VMX_VMCS_GUEST_DS_BASE:
6420	case VMX_VMCS_GUEST_FS_BASE:
6421	case VMX_VMCS_GUEST_GS_BASE:
6422	case VMX_VMCS_GUEST_LDTR_BASE:
6423	case VMX_VMCS_GUEST_TR_BASE:
6424	case VMX_VMCS_GUEST_GDTR_BASE:
6425	case VMX_VMCS_GUEST_IDTR_BASE:
6426	case VMX_VMCS_GUEST_RSP:
6427	case VMX_VMCS_GUEST_RIP:
6428	case VMX_VMCS_GUEST_SYSENTER_ESP:
6429	case VMX_VMCS_GUEST_SYSENTER_EIP:
6430	{
6431	if (!(RT_HI_U32(u64Val)))
6432	{
6433	/* If this field is 64-bit, VT-x will zero out the top bits. */
6434	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6435	}
6436	else
6437	{
6438	/* Assert that only the 32->64 switcher case should ever come here. */
6439	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
6440	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
6441	}
6442	break;
6443	}
6444
6445	default:
6446	{
6447	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
6448	pVCpu->hm.s.u32HMError = idxField;
6449	rc = VERR_INVALID_PARAMETER;
6450	break;
6451	}
6452	}
6453	AssertRCReturn(rc, rc);
6454	return rc;
6455	}
6456
6457
6458	/**
6459	* Queue up a VMWRITE by using the VMCS write cache.
6460	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
6461	*
6462	* @param pVCpu The cross context virtual CPU structure.
6463	* @param idxField The VMCS field encoding.
6464	* @param u64Val 16, 32 or 64-bit value.
6465	*/
6466	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6467	{
6468	AssertPtr(pVCpu);
6469	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6470
6471	AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
6472	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
6473
6474	/* Make sure there are no duplicates. */
6475	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
6476	{
6477	if (pCache->Write.aField[i] == idxField)
6478	{
6479	pCache->Write.aFieldVal[i] = u64Val;
6480	return VINF_SUCCESS;
6481	}
6482	}
6483
6484	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
6485	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
6486	pCache->Write.cValidEntries++;
6487	return VINF_SUCCESS;
6488	}
6489	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
6490
6491
6492	/**
6493	* Sets up the usage of TSC-offsetting and updates the VMCS.
6494	*
6495	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6496	* VMX-preemption timer.
6497	*
6498	* @returns VBox status code.
6499	* @param pVCpu The cross context virtual CPU structure.
6500	* @param pVmxTransient The VMX-transient structure.
6501	*
6502	* @remarks No-long-jump zone!!!
6503	*/
6504	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6505	{
6506	bool fOffsettedTsc;
6507	bool fParavirtTsc;
6508	uint64_t uTscOffset;
6509	PVM pVM = pVCpu->CTX_SUFF(pVM);
6510	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6511
6512	if (pVM->hm.s.vmx.fUsePreemptTimer)
6513	{
6514	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6515
6516	/* Make sure the returned values have sane upper and lower boundaries. */
6517	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6518	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6519	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6520	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6521
6522	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6523	* preemption timers here. We probably need to clamp the preemption timer,
6524	* after converting the timer value to the host. */
6525	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6526	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6527	AssertRC(rc);
6528	}
6529	else
6530	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6531
6532	if (fParavirtTsc)
6533	{
6534	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6535	information before every VM-entry, hence disable it for performance sake. */
6536	#if 0
6537	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
6538	AssertRC(rc);
6539	#endif
6540	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
6541	}
6542
6543	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
6544	if ( fOffsettedTsc
6545	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
6546	{
6547	if (pVmxTransient->fIsNestedGuest)
6548	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
6549	if (pVmcsInfo->u64TscOffset != uTscOffset)
6550	{
6551	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
6552	AssertRC(rc);
6553	pVmcsInfo->u64TscOffset = uTscOffset;
6554	}
6555
6556	if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
6557	{
6558	uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
6559	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6560	AssertRC(rc);
6561	pVmcsInfo->u32ProcCtls = uProcCtls;
6562	}
6563	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
6564	}
6565	else
6566	{
6567	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
6568	if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
6569	{
6570	uProcCtls \|= VMX_PROC_CTLS_RDTSC_EXIT;
6571	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6572	AssertRC(rc);
6573	pVmcsInfo->u32ProcCtls = uProcCtls;
6574	}
6575	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
6576	}
6577	}
6578
6579
6580	/**
6581	* Gets the IEM exception flags for the specified vector and IDT vectoring /
6582	* VM-exit interruption info type.
6583	*
6584	* @returns The IEM exception flags.
6585	* @param uVector The event vector.
6586	* @param uVmxEventType The VMX event type.
6587	*
6588	* @remarks This function currently only constructs flags required for
6589	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
6590	* and CR2 aspects of an exception are not included).
6591	*/
6592	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
6593	{
6594	uint32_t fIemXcptFlags;
6595	switch (uVmxEventType)
6596	{
6597	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6598	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6599	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6600	break;
6601
6602	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6603	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6604	break;
6605
6606	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6607	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
6608	break;
6609
6610	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
6611	{
6612	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6613	if (uVector == X86_XCPT_BP)
6614	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
6615	else if (uVector == X86_XCPT_OF)
6616	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
6617	else
6618	{
6619	fIemXcptFlags = 0;
6620	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
6621	}
6622	break;
6623	}
6624
6625	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6626	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6627	break;
6628
6629	default:
6630	fIemXcptFlags = 0;
6631	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
6632	break;
6633	}
6634	return fIemXcptFlags;
6635	}
6636
6637
6638	/**
6639	* Sets an event as a pending event to be injected into the guest.
6640	*
6641	* @param pVCpu The cross context virtual CPU structure.
6642	* @param u32IntInfo The VM-entry interruption-information field.
6643	* @param cbInstr The VM-entry instruction length in bytes (for software
6644	* interrupts, exceptions and privileged software
6645	* exceptions).
6646	* @param u32ErrCode The VM-entry exception error code.
6647	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
6648	* page-fault.
6649	*/
6650	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
6651	RTGCUINTPTR GCPtrFaultAddress)
6652	{
6653	Assert(!pVCpu->hm.s.Event.fPending);
6654	pVCpu->hm.s.Event.fPending = true;
6655	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
6656	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
6657	pVCpu->hm.s.Event.cbInstr = cbInstr;
6658	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
6659	}
6660
6661
6662	/**
6663	* Sets an external interrupt as pending-for-injection into the VM.
6664	*
6665	* @param pVCpu The cross context virtual CPU structure.
6666	* @param u8Interrupt The external interrupt vector.
6667	*/
6668	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
6669	{
6670	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
6671	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6672	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6673	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6674	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6675	}
6676
6677
6678	/**
6679	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
6680	*
6681	* @param pVCpu The cross context virtual CPU structure.
6682	*/
6683	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
6684	{
6685	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
6686	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
6687	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6688	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6689	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6690	}
6691
6692
6693	/**
6694	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
6695	*
6696	* @param pVCpu The cross context virtual CPU structure.
6697	*/
6698	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
6699	{
6700	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
6701	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6702	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6703	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6704	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6705	}
6706
6707
6708	/**
6709	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
6710	*
6711	* @param pVCpu The cross context virtual CPU structure.
6712	*/
6713	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
6714	{
6715	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
6716	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6717	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6718	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6719	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6720	}
6721
6722
6723	/**
6724	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
6725	*
6726	* @param pVCpu The cross context virtual CPU structure.
6727	*/
6728	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
6729	{
6730	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
6731	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6732	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6733	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6734	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6735	}
6736
6737
6738	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6739	/**
6740	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
6741	*
6742	* @param pVCpu The cross context virtual CPU structure.
6743	* @param u32ErrCode The error code for the general-protection exception.
6744	*/
6745	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
6746	{
6747	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
6748	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6749	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6750	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6751	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6752	}
6753
6754
6755	/**
6756	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
6757	*
6758	* @param pVCpu The cross context virtual CPU structure.
6759	* @param u32ErrCode The error code for the stack exception.
6760	*/
6761	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
6762	{
6763	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
6764	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6765	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6766	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6767	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6768	}
6769
6770
6771	/**
6772	* Decodes the memory operand of an instruction that caused a VM-exit.
6773	*
6774	* The VM-exit qualification field provides the displacement field for memory
6775	* operand instructions, if any.
6776	*
6777	* @returns Strict VBox status code (i.e. informational status codes too).
6778	* @retval VINF_SUCCESS if the operand was successfully decoded.
6779	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
6780	* operand.
6781	* @param pVCpu The cross context virtual CPU structure.
6782	* @param uExitInstrInfo The VM-exit instruction information field.
6783	* @param enmMemAccess The memory operand's access type (read or write).
6784	* @param GCPtrDisp The instruction displacement field, if any. For
6785	* RIP-relative addressing pass RIP + displacement here.
6786	* @param pGCPtrMem Where to store the effective destination memory address.
6787	*/
6788	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
6789	PRTGCPTR pGCPtrMem)
6790	{
6791	Assert(pGCPtrMem);
6792	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
6793	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
6794	\| CPUMCTX_EXTRN_CR0);
6795
6796	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
6797	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
6798	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
6799
6800	VMXEXITINSTRINFO ExitInstrInfo;
6801	ExitInstrInfo.u = uExitInstrInfo;
6802	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
6803	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
6804	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
6805	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
6806	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
6807	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
6808	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
6809	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
6810	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
6811
6812	/*
6813	* Validate instruction information.
6814	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
6815	*/
6816	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
6817	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
6818	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
6819	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
6820	AssertLogRelMsgReturn(fIsMemOperand,
6821	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
6822
6823	/*
6824	* Compute the complete effective address.
6825	*
6826	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
6827	* See AMD spec. 4.5.2 "Segment Registers".
6828	*/
6829	RTGCPTR GCPtrMem = GCPtrDisp;
6830	if (fBaseRegValid)
6831	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
6832	if (fIdxRegValid)
6833	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
6834
6835	RTGCPTR const GCPtrOff = GCPtrMem;
6836	if ( !fIsLongMode
6837	\|\| iSegReg >= X86_SREG_FS)
6838	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
6839	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
6840
6841	/*
6842	* Validate effective address.
6843	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
6844	*/
6845	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
6846	Assert(cbAccess > 0);
6847	if (fIsLongMode)
6848	{
6849	if (X86_IS_CANONICAL(GCPtrMem))
6850	{
6851	*pGCPtrMem = GCPtrMem;
6852	return VINF_SUCCESS;
6853	}
6854
6855	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
6856	* "Data Limit Checks in 64-bit Mode". */
6857	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
6858	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6859	return VINF_HM_PENDING_XCPT;
6860	}
6861
6862	/*
6863	* This is a watered down version of iemMemApplySegment().
6864	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
6865	* and segment CPL/DPL checks are skipped.
6866	*/
6867	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
6868	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
6869	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
6870
6871	/* Check if the segment is present and usable. */
6872	if ( pSel->Attr.n.u1Present
6873	&& !pSel->Attr.n.u1Unusable)
6874	{
6875	Assert(pSel->Attr.n.u1DescType);
6876	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
6877	{
6878	/* Check permissions for the data segment. */
6879	if ( enmMemAccess == VMXMEMACCESS_WRITE
6880	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
6881	{
6882	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6883	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
6884	return VINF_HM_PENDING_XCPT;
6885	}
6886
6887	/* Check limits if it's a normal data segment. */
6888	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
6889	{
6890	if ( GCPtrFirst32 > pSel->u32Limit
6891	\|\| GCPtrLast32 > pSel->u32Limit)
6892	{
6893	Log4Func(("Data segment limit exceeded."
6894	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6895	GCPtrLast32, pSel->u32Limit));
6896	if (iSegReg == X86_SREG_SS)
6897	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6898	else
6899	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6900	return VINF_HM_PENDING_XCPT;
6901	}
6902	}
6903	else
6904	{
6905	/* Check limits if it's an expand-down data segment.
6906	Note! The upper boundary is defined by the B bit, not the G bit! */
6907	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
6908	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
6909	{
6910	Log4Func(("Expand-down data segment limit exceeded."
6911	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6912	GCPtrLast32, pSel->u32Limit));
6913	if (iSegReg == X86_SREG_SS)
6914	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6915	else
6916	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6917	return VINF_HM_PENDING_XCPT;
6918	}
6919	}
6920	}
6921	else
6922	{
6923	/* Check permissions for the code segment. */
6924	if ( enmMemAccess == VMXMEMACCESS_WRITE
6925	\|\| ( enmMemAccess == VMXMEMACCESS_READ
6926	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
6927	{
6928	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6929	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6930	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6931	return VINF_HM_PENDING_XCPT;
6932	}
6933
6934	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6935	if ( GCPtrFirst32 > pSel->u32Limit
6936	\|\| GCPtrLast32 > pSel->u32Limit)
6937	{
6938	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6939	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6940	if (iSegReg == X86_SREG_SS)
6941	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6942	else
6943	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6944	return VINF_HM_PENDING_XCPT;
6945	}
6946	}
6947	}
6948	else
6949	{
6950	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6951	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6952	return VINF_HM_PENDING_XCPT;
6953	}
6954
6955	*pGCPtrMem = GCPtrMem;
6956	return VINF_SUCCESS;
6957	}
6958
6959
6960	/**
6961	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6962	* guest attempting to execute a VMX instruction.
6963	*
6964	* @returns Strict VBox status code (i.e. informational status codes too).
6965	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
6966	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
6967	*
6968	* @param pVCpu The cross context virtual CPU structure.
6969	* @param uExitReason The VM-exit reason.
6970	*
6971	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
6972	* @remarks No-long-jump zone!!!
6973	*/
6974	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
6975	{
6976	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
6977	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
6978
6979	if ( CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx)
6980	\|\| ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6981	&& !CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)))
6982	{
6983	Log4Func(("In real/v86-mode or long-mode outside 64-bit code segment -> #UD\n"));
6984	hmR0VmxSetPendingXcptUD(pVCpu);
6985	return VINF_HM_PENDING_XCPT;
6986	}
6987
6988	if (uExitReason == VMX_EXIT_VMXON)
6989	{
6990	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
6991
6992	/*
6993	* We check CR4.VMXE because it is required to be always set while in VMX operation
6994	* by physical CPUs and our CR4 read shadow is only consulted when executing specific
6995	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
6996	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
6997	*/
6998	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
6999	{
7000	Log4Func(("CR4.VMXE is not set -> #UD\n"));
7001	hmR0VmxSetPendingXcptUD(pVCpu);
7002	return VINF_HM_PENDING_XCPT;
7003	}
7004	}
7005	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
7006	{
7007	/*
7008	* The guest has not entered VMX operation but attempted to execute a VMX instruction
7009	* (other than VMXON), we need to raise a #UD.
7010	*/
7011	Log4Func(("Not in VMX root mode -> #UD\n"));
7012	hmR0VmxSetPendingXcptUD(pVCpu);
7013	return VINF_HM_PENDING_XCPT;
7014	}
7015
7016	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
7017	{
7018	/*
7019	* The nested-guest attempted to execute a VMX instruction, cause a VM-exit and let
7020	* the guest hypervisor deal with it.
7021	*/
7022	/** @todo NSTVMX: Trigger a VM-exit */
7023	}
7024
7025	/*
7026	* VMX instructions require CPL 0 except in VMX non-root mode where the VM-exit intercept
7027	* (above) takes preceedence over the CPL check.
7028	*/
7029	if (CPUMGetGuestCPL(pVCpu) > 0)
7030	{
7031	Log4Func(("CPL > 0 -> #GP(0)\n"));
7032	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7033	return VINF_HM_PENDING_XCPT;
7034	}
7035
7036	return VINF_SUCCESS;
7037	}
7038	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7039
7040
7041	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7042	{
7043	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7044
7045	/*
7046	* If VT-x marks the segment as unusable, most other bits remain undefined:
7047	* - For CS the L, D and G bits have meaning.
7048	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7049	* - For the remaining data segments no bits are defined.
7050	*
7051	* The present bit and the unusable bit has been observed to be set at the
7052	* same time (the selector was supposed to be invalid as we started executing
7053	* a V8086 interrupt in ring-0).
7054	*
7055	* What should be important for the rest of the VBox code, is that the P bit is
7056	* cleared. Some of the other VBox code recognizes the unusable bit, but
7057	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7058	* safe side here, we'll strip off P and other bits we don't care about. If
7059	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7060	*
7061	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7062	*/
7063	#ifdef VBOX_STRICT
7064	uint32_t const uAttr = pSelReg->Attr.u;
7065	#endif
7066
7067	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7068	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7069	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7070
7071	#ifdef VBOX_STRICT
7072	VMMRZCallRing3Disable(pVCpu);
7073	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7074	# ifdef DEBUG_bird
7075	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7076	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7077	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7078	# endif
7079	VMMRZCallRing3Enable(pVCpu);
7080	NOREF(uAttr);
7081	#endif
7082	RT_NOREF2(pVCpu, idxSel);
7083	}
7084
7085
7086	/**
7087	* Imports a guest segment register from the current VMCS into the guest-CPU
7088	* context.
7089	*
7090	* @returns VBox status code.
7091	* @param pVCpu The cross context virtual CPU structure.
7092	* @param iSegReg The segment register number (X86_SREG_XXX).
7093	*
7094	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7095	* do not log!
7096	*/
7097	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7098	{
7099	Assert(iSegReg < X86_SREG_COUNT);
7100
7101	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7102	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7103	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7104	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
7105	uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
7106	#else
7107	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7108	#endif
7109	uint64_t u64Base;
7110	uint32_t u32Sel, u32Limit, u32Attr;
7111	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7112	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7113	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7114	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
7115	if (RT_SUCCESS(rc))
7116	{
7117	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7118	pSelReg->Sel = u32Sel;
7119	pSelReg->ValidSel = u32Sel;
7120	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7121	pSelReg->u32Limit = u32Limit;
7122	pSelReg->u64Base = u64Base;
7123	pSelReg->Attr.u = u32Attr;
7124	if (u32Attr & X86DESCATTR_UNUSABLE)
7125	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7126	}
7127	return rc;
7128	}
7129
7130
7131	/**
7132	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7133	*
7134	* @returns VBox status code.
7135	* @param pVCpu The cross context virtual CPU structure.
7136	*
7137	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7138	* do not log!
7139	*/
7140	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7141	{
7142	uint64_t u64Base;
7143	uint32_t u32Sel, u32Limit, u32Attr;
7144	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7145	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7146	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7147	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7148	if (RT_SUCCESS(rc))
7149	{
7150	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7151	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7152	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7153	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7154	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7155	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7156	if (u32Attr & X86DESCATTR_UNUSABLE)
7157	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7158	}
7159	return rc;
7160	}
7161
7162
7163	/**
7164	* Imports the guest TR from the current VMCS into the guest-CPU context.
7165	*
7166	* @returns VBox status code.
7167	* @param pVCpu The cross context virtual CPU structure.
7168	*
7169	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7170	* do not log!
7171	*/
7172	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7173	{
7174	uint32_t u32Sel, u32Limit, u32Attr;
7175	uint64_t u64Base;
7176	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7177	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7178	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7179	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7180	AssertRCReturn(rc, rc);
7181
7182	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7183	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7184	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7185	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7186	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7187	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7188	/* TR is the only selector that can never be unusable. */
7189	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7190	return VINF_SUCCESS;
7191	}
7192
7193
7194	/**
7195	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7196	*
7197	* @returns VBox status code.
7198	* @param pVCpu The cross context virtual CPU structure.
7199	*
7200	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7201	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7202	* instead!!!
7203	*/
7204	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7205	{
7206	uint64_t u64Val;
7207	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7208	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7209	{
7210	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7211	if (RT_SUCCESS(rc))
7212	{
7213	pCtx->rip = u64Val;
7214	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7215	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7216	}
7217	return rc;
7218	}
7219	return VINF_SUCCESS;
7220	}
7221
7222
7223	/**
7224	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7225	*
7226	* @returns VBox status code.
7227	* @param pVCpu The cross context virtual CPU structure.
7228	* @param pVmcsInfo The VMCS info. object.
7229	*
7230	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7231	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7232	* instead!!!
7233	*/
7234	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7235	{
7236	uint32_t u32Val;
7237	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7238	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7239	{
7240	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7241	if (RT_SUCCESS(rc))
7242	{
7243	pCtx->eflags.u32 = u32Val;
7244
7245	/* Restore eflags for real-on-v86-mode hack. */
7246	if (pVmcsInfo->RealMode.fRealOnV86Active)
7247	{
7248	pCtx->eflags.Bits.u1VM = 0;
7249	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7250	}
7251	}
7252	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7253	return rc;
7254	}
7255	return VINF_SUCCESS;
7256	}
7257
7258
7259	/**
7260	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7261	* context.
7262	*
7263	* @returns VBox status code.
7264	* @param pVCpu The cross context virtual CPU structure.
7265	* @param pVmcsInfo The VMCS info. object.
7266	*
7267	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7268	* do not log!
7269	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7270	* instead!!!
7271	*/
7272	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7273	{
7274	uint32_t u32Val;
7275	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7276	if (RT_SUCCESS(rc))
7277	{
7278	if (!u32Val)
7279	{
7280	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7281	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7282
7283	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7284	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7285	}
7286	else
7287	{
7288	/*
7289	* We must import RIP here to set our EM interrupt-inhibited state.
7290	* We also import RFLAGS as our code that evaluates pending interrupts
7291	* before VM-entry requires it.
7292	*/
7293	rc = hmR0VmxImportGuestRip(pVCpu);
7294	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7295	if (RT_SUCCESS(rc))
7296	{
7297	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7298	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7299	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7300	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7301
7302	if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
7303	{
7304	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7305	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
7306	}
7307	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7308	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7309	}
7310	}
7311	}
7312	return rc;
7313	}
7314
7315
7316	/**
7317	* Worker for VMXR0ImportStateOnDemand.
7318	*
7319	* @returns VBox status code.
7320	* @param pVCpu The cross context virtual CPU structure.
7321	* @param pVmcsInfo The VMCS info. object.
7322	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7323	*/
7324	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7325	{
7326	#define VMXLOCAL_BREAK_RC(a_rc) \
7327	if (RT_SUCCESS(a_rc)) \
7328	{ } \
7329	else \
7330	break
7331
7332	int rc = VINF_SUCCESS;
7333	PVM pVM = pVCpu->CTX_SUFF(pVM);
7334	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7335	uint64_t u64Val;
7336	uint32_t u32Val;
7337
7338	/*
7339	* Note! This is hack to workaround a mysterious BSOD observed with release builds
7340	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
7341	* neither are other host platforms.
7342	*
7343	* Committing this temporarily as it prevents BSOD.
7344	*
7345	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
7346	*/
7347	#ifdef RT_OS_WINDOWS
7348	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
7349	return VERR_HM_IPE_1;
7350	#endif
7351
7352	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
7353
7354	/*
7355	* We disable interrupts to make the updating of the state and in particular
7356	* the fExtrn modification atomic wrt to preemption hooks.
7357	*/
7358	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7359
7360	fWhat &= pCtx->fExtrn;
7361	if (fWhat)
7362	{
7363	do
7364	{
7365	if (fWhat & CPUMCTX_EXTRN_RIP)
7366	{
7367	rc = hmR0VmxImportGuestRip(pVCpu);
7368	VMXLOCAL_BREAK_RC(rc);
7369	}
7370
7371	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7372	{
7373	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7374	VMXLOCAL_BREAK_RC(rc);
7375	}
7376
7377	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7378	{
7379	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7380	VMXLOCAL_BREAK_RC(rc);
7381	}
7382
7383	if (fWhat & CPUMCTX_EXTRN_RSP)
7384	{
7385	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
7386	VMXLOCAL_BREAK_RC(rc);
7387	pCtx->rsp = u64Val;
7388	}
7389
7390	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7391	{
7392	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7393	if (fWhat & CPUMCTX_EXTRN_CS)
7394	{
7395	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7396	rc \|= hmR0VmxImportGuestRip(pVCpu);
7397	if (fRealOnV86Active)
7398	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7399	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7400	}
7401	if (fWhat & CPUMCTX_EXTRN_SS)
7402	{
7403	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7404	if (fRealOnV86Active)
7405	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7406	}
7407	if (fWhat & CPUMCTX_EXTRN_DS)
7408	{
7409	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7410	if (fRealOnV86Active)
7411	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7412	}
7413	if (fWhat & CPUMCTX_EXTRN_ES)
7414	{
7415	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7416	if (fRealOnV86Active)
7417	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7418	}
7419	if (fWhat & CPUMCTX_EXTRN_FS)
7420	{
7421	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7422	if (fRealOnV86Active)
7423	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7424	}
7425	if (fWhat & CPUMCTX_EXTRN_GS)
7426	{
7427	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7428	if (fRealOnV86Active)
7429	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7430	}
7431	VMXLOCAL_BREAK_RC(rc);
7432	}
7433
7434	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7435	{
7436	if (fWhat & CPUMCTX_EXTRN_LDTR)
7437	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
7438
7439	if (fWhat & CPUMCTX_EXTRN_GDTR)
7440	{
7441	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
7442	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
7443	pCtx->gdtr.pGdt = u64Val;
7444	pCtx->gdtr.cbGdt = u32Val;
7445	}
7446
7447	/* Guest IDTR. */
7448	if (fWhat & CPUMCTX_EXTRN_IDTR)
7449	{
7450	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
7451	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
7452	pCtx->idtr.pIdt = u64Val;
7453	pCtx->idtr.cbIdt = u32Val;
7454	}
7455
7456	/* Guest TR. */
7457	if (fWhat & CPUMCTX_EXTRN_TR)
7458	{
7459	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7460	don't need to import that one. */
7461	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7462	rc \|= hmR0VmxImportGuestTr(pVCpu);
7463	}
7464	VMXLOCAL_BREAK_RC(rc);
7465	}
7466
7467	if (fWhat & CPUMCTX_EXTRN_DR7)
7468	{
7469	if (!pVCpu->hm.s.fUsingHyperDR7)
7470	{
7471	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
7472	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
7473	VMXLOCAL_BREAK_RC(rc);
7474	pCtx->dr[7] = u32Val;
7475	}
7476	}
7477
7478	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7479	{
7480	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
7481	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
7482	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
7483	pCtx->SysEnter.cs = u32Val;
7484	VMXLOCAL_BREAK_RC(rc);
7485	}
7486
7487	#if HC_ARCH_BITS == 64
7488	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7489	{
7490	if ( pVM->hm.s.fAllow64BitGuests
7491	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7492	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7493	}
7494
7495	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7496	{
7497	if ( pVM->hm.s.fAllow64BitGuests
7498	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7499	{
7500	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7501	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7502	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7503	}
7504	}
7505	#endif
7506
7507	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7508	#if HC_ARCH_BITS == 32
7509	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
7510	#endif
7511	)
7512	{
7513	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7514	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7515	Assert(pMsrs);
7516	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
7517	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
7518	for (uint32_t i = 0; i < cMsrs; i++)
7519	{
7520	uint32_t const idMsr = pMsrs[i].u32Msr;
7521	switch (idMsr)
7522	{
7523	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
7524	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
7525	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7526	#if HC_ARCH_BITS == 32
7527	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsrs[i].u64Value; break;
7528	case MSR_K6_STAR: pCtx->msrSTAR = pMsrs[i].u64Value; break;
7529	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsrs[i].u64Value; break;
7530	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsrs[i].u64Value; break;
7531	#endif
7532	default:
7533	{
7534	pCtx->fExtrn = 0;
7535	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
7536	ASMSetFlags(fEFlags);
7537	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
7538	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7539	}
7540	}
7541	}
7542	}
7543
7544	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7545	{
7546	uint64_t u64Shadow;
7547	if (fWhat & CPUMCTX_EXTRN_CR0)
7548	{
7549	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7550	* remove when we drop 32-bit host w/ 64-bit host support, see
7551	* @bugref{9180#c39}. */
7552	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7553	#if HC_ARCH_BITS == 32
7554	uint32_t u32Shadow;
7555	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
7556	u64Shadow = u32Shadow;
7557	#else
7558	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
7559	#endif
7560	VMXLOCAL_BREAK_RC(rc);
7561	u64Val = u32Val;
7562	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7563	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7564	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7565	CPUMSetGuestCR0(pVCpu, u64Val);
7566	VMMRZCallRing3Enable(pVCpu);
7567	}
7568
7569	if (fWhat & CPUMCTX_EXTRN_CR4)
7570	{
7571	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7572	* remove when we drop 32-bit host w/ 64-bit host support, see
7573	* @bugref{9180#c39}. */
7574	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7575	#if HC_ARCH_BITS == 32
7576	uint32_t u32Shadow;
7577	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
7578	u64Shadow = u32Shadow;
7579	#else
7580	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
7581	#endif
7582	VMXLOCAL_BREAK_RC(rc);
7583	u64Val = u32Val;
7584	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
7585	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7586	pCtx->cr4 = u64Val;
7587	}
7588
7589	if (fWhat & CPUMCTX_EXTRN_CR3)
7590	{
7591	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7592	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7593	\|\| ( pVM->hm.s.fNestedPaging
7594	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7595	{
7596	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
7597	VMXLOCAL_BREAK_RC(rc);
7598	if (pCtx->cr3 != u64Val)
7599	{
7600	pCtx->cr3 = u64Val;
7601	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7602	}
7603
7604	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7605	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7606	if (CPUMIsGuestInPAEModeEx(pCtx))
7607	{
7608	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
7609	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
7610	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
7611	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
7612	VMXLOCAL_BREAK_RC(rc);
7613	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7614	}
7615	}
7616	}
7617
7618	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7619	# if 0
7620	/** @todo NSTVMX: We handle each of these fields individually by passing it to IEM
7621	* VM-exit handlers. We might handle it differently when using the fast path. */
7622	/*
7623	* The hardware virtualization state currently consists of VMCS fields that may be
7624	* modified by execution of the nested-guest (that are not part of the general
7625	* guest state) and is visible to guest software. Hence, it is technically part of
7626	* the guest-CPU state when executing a nested-guest.
7627	*/
7628	if ( (fWhat & CPUMCTX_EXTRN_HWVIRT)
7629	&& CPUMIsGuestInVmxNonRootMode(pCtx))
7630	{
7631	PVMXVVMCS pGstVmcs = pCtx->hwvirt.vmx.CTX_SUFF(pVmcs);
7632	rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pGstVmcs->u32RoExitReason);
7633	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pGstVmcs->u64RoExitQual.u);
7634	VMXLOCAL_BREAK_RC(rc);
7635
7636	/*
7637	* VM-entry can fail due to invalid-guest state, machine-check events and
7638	* MSR loading failures. Other than VM-exit reason and VM-exit qualification
7639	* all other VMCS fields are left unmodified on VM-entry failure.
7640	*
7641	* See Intel spec. 26.7 "VM-entry Failures During Or After Loading Guest State".
7642	*/
7643	bool const fEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(pGstVmcs->u32RoExitReason);
7644	if (!fEntryFailed)
7645	{
7646	/*
7647	* Some notes on VMCS fields that may need importing when the fast path
7648	* is implemented. Currently we fully emulate VMLAUNCH/VMRESUME in IEM.
7649	*
7650	* Requires fixing up when using hardware-assisted VMX:
7651	* - VM-exit interruption info: Shouldn't reflect host interrupts/NMIs.
7652	* - VM-exit interruption error code: Cleared to 0 when not appropriate.
7653	* - IDT-vectoring info: Think about this.
7654	* - IDT-vectoring error code: Think about this.
7655	*
7656	* Emulated:
7657	* - Guest-interruptiblity state: Derived from FFs and RIP.
7658	* - Guest pending debug exceptions: Derived from DR6.
7659	* - Guest activity state: Emulated from EM state.
7660	* - Guest PDPTEs: Currently all 0s since we don't support nested EPT.
7661	* - Entry-interrupt info: Emulated, cleared to 0.
7662	*/
7663	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pGstVmcs->u32RoExitIntInfo);
7664	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pGstVmcs->u32RoExitIntErrCode);
7665	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pGstVmcs->u32RoIdtVectoringInfo);
7666	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pGstVmcs->u32RoIdtVectoringErrCode);
7667	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pGstVmcs->u32RoExitInstrLen);
7668	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pGstVmcs->u32RoExitIntInfo);
7669	rc \|= VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pGstVmcs->u64RoGuestPhysAddr.u);
7670	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pGstVmcs->u64RoGuestLinearAddr.u);
7671	/** @todo NSTVMX: Save and adjust preemption timer value. */
7672	}
7673
7674	VMXLOCAL_BREAK_RC(rc);
7675	}
7676	# endif
7677	#endif
7678	}
7679	} while (0);
7680
7681	if (RT_SUCCESS(rc))
7682	{
7683	/* Update fExtrn. */
7684	pCtx->fExtrn &= ~fWhat;
7685
7686	/* If everything has been imported, clear the HM keeper bit. */
7687	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7688	{
7689	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7690	Assert(!pCtx->fExtrn);
7691	}
7692	}
7693	}
7694	else
7695	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7696
7697	ASMSetFlags(fEFlags);
7698
7699	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7700
7701	if (RT_SUCCESS(rc))
7702	{ /* likely */ }
7703	else
7704	return rc;
7705
7706	/*
7707	* Honor any pending CR3 updates.
7708	*
7709	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
7710	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7711	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7712	*
7713	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7714	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7715	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7716	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7717	*
7718	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7719	*/
7720	if (VMMRZCallRing3IsEnabled(pVCpu))
7721	{
7722	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7723	{
7724	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7725	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7726	}
7727
7728	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7729	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7730
7731	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7732	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7733	}
7734
7735	return VINF_SUCCESS;
7736	#undef VMXLOCAL_BREAK_RC
7737	}
7738
7739
7740	/**
7741	* Saves the guest state from the VMCS into the guest-CPU context.
7742	*
7743	* @returns VBox status code.
7744	* @param pVCpu The cross context virtual CPU structure.
7745	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7746	*/
7747	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
7748	{
7749	PCVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7750	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7751	}
7752
7753
7754	/**
7755	* Check per-VM and per-VCPU force flag actions that require us to go back to
7756	* ring-3 for one reason or another.
7757	*
7758	* @returns Strict VBox status code (i.e. informational status codes too)
7759	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7760	* ring-3.
7761	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7762	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7763	* interrupts)
7764	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7765	* all EMTs to be in ring-3.
7766	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7767	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7768	* to the EM loop.
7769	*
7770	* @param pVCpu The cross context virtual CPU structure.
7771	* @param fStepping Whether we are single-stepping the guest using the
7772	* hypervisor debugger.
7773	*/
7774	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
7775	{
7776	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7777
7778	/*
7779	* Update pending interrupts into the APIC's IRR.
7780	*/
7781	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7782	APICUpdatePendingInterrupts(pVCpu);
7783
7784	/*
7785	* Anything pending? Should be more likely than not if we're doing a good job.
7786	*/
7787	PVM pVM = pVCpu->CTX_SUFF(pVM);
7788	if ( !fStepping
7789	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7790	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7791	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7792	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7793	return VINF_SUCCESS;
7794
7795	/* Pending PGM C3 sync. */
7796	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7797	{
7798	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7799	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7800	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7801	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7802	if (rcStrict2 != VINF_SUCCESS)
7803	{
7804	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7805	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7806	return rcStrict2;
7807	}
7808	}
7809
7810	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7811	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7812	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7813	{
7814	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7815	int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7816	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7817	return rc2;
7818	}
7819
7820	/* Pending VM request packets, such as hardware interrupts. */
7821	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7822	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7823	{
7824	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7825	return VINF_EM_PENDING_REQUEST;
7826	}
7827
7828	/* Pending PGM pool flushes. */
7829	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7830	{
7831	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7832	return VINF_PGM_POOL_FLUSH_PENDING;
7833	}
7834
7835	/* Pending DMA requests. */
7836	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7837	{
7838	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7839	return VINF_EM_RAW_TO_R3;
7840	}
7841
7842	return VINF_SUCCESS;
7843	}
7844
7845
7846	/**
7847	* Converts any TRPM trap into a pending HM event. This is typically used when
7848	* entering from ring-3 (not longjmp returns).
7849	*
7850	* @param pVCpu The cross context virtual CPU structure.
7851	*/
7852	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7853	{
7854	Assert(TRPMHasTrap(pVCpu));
7855	Assert(!pVCpu->hm.s.Event.fPending);
7856
7857	uint8_t uVector;
7858	TRPMEVENT enmTrpmEvent;
7859	RTGCUINT uErrCode;
7860	RTGCUINTPTR GCPtrFaultAddress;
7861	uint8_t cbInstr;
7862
7863	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7864	AssertRC(rc);
7865
7866	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7867	uint32_t u32IntInfo = uVector \| VMX_EXIT_INT_INFO_VALID;
7868	if (enmTrpmEvent == TRPM_TRAP)
7869	{
7870	/** @todo r=ramshankar: TRPM currently offers no way to determine a \#DB that was
7871	* generated using INT1 (ICEBP). */
7872	switch (uVector)
7873	{
7874	case X86_XCPT_NMI:
7875	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7876	break;
7877
7878	case X86_XCPT_BP:
7879	case X86_XCPT_OF:
7880	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7881	break;
7882
7883	case X86_XCPT_PF:
7884	case X86_XCPT_DF:
7885	case X86_XCPT_TS:
7886	case X86_XCPT_NP:
7887	case X86_XCPT_SS:
7888	case X86_XCPT_GP:
7889	case X86_XCPT_AC:
7890	u32IntInfo \|= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7891	RT_FALL_THRU();
7892	default:
7893	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7894	break;
7895	}
7896	}
7897	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7898	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7899	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7900	{
7901	switch (uVector)
7902	{
7903	case X86_XCPT_BP:
7904	case X86_XCPT_OF:
7905	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7906	break;
7907
7908	default:
7909	Assert(uVector == X86_XCPT_DB);
7910	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7911	break;
7912	}
7913	}
7914	else
7915	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7916
7917	rc = TRPMResetTrap(pVCpu);
7918	AssertRC(rc);
7919	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7920	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7921
7922	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7923	}
7924
7925
7926	/**
7927	* Converts the pending HM event into a TRPM trap.
7928	*
7929	* @param pVCpu The cross context virtual CPU structure.
7930	*/
7931	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7932	{
7933	Assert(pVCpu->hm.s.Event.fPending);
7934
7935	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7936	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7937	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
7938	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7939
7940	/* If a trap was already pending, we did something wrong! */
7941	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7942
7943	/** @todo Use HMVmxEventToTrpmEventType() later. */
7944	TRPMEVENT enmTrapType;
7945	switch (uVectorType)
7946	{
7947	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7948	enmTrapType = TRPM_HARDWARE_INT;
7949	break;
7950
7951	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7952	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7953	enmTrapType = TRPM_TRAP;
7954	break;
7955
7956	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT: /* #DB (INT1/ICEBP). */
7957	Assert(uVector == X86_XCPT_DB);
7958	enmTrapType = TRPM_SOFTWARE_INT;
7959	break;
7960
7961	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP (INT3) and #OF (INTO) */
7962	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7963	enmTrapType = TRPM_SOFTWARE_INT;
7964	break;
7965
7966	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7967	enmTrapType = TRPM_SOFTWARE_INT;
7968	break;
7969
7970	default:
7971	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7972	enmTrapType = TRPM_32BIT_HACK;
7973	break;
7974	}
7975
7976	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7977
7978	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7979	AssertRC(rc);
7980
7981	if (fErrorCodeValid)
7982	TRPMSetErrorCode(pVCpu, uErrorCode);
7983
7984	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7985	&& uVector == X86_XCPT_PF)
7986	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7987	else if (enmTrapType == TRPM_SOFTWARE_INT)
7988	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7989
7990	/* We're now done converting the pending event. */
7991	pVCpu->hm.s.Event.fPending = false;
7992	}
7993
7994
7995	/**
7996	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7997	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7998	*
7999	* @param pVCpu The cross context virtual CPU structure.
8000	* @param pVmcsInfo The VMCS info. object.
8001	*/
8002	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8003	{
8004	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8005	{
8006	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
8007	{
8008	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
8009	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8010	AssertRC(rc);
8011	}
8012	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
8013	}
8014
8015
8016	/**
8017	* Clears the interrupt-window exiting control in the VMCS.
8018	*
8019	* @param pVmcsInfo The VMCS info. object.
8020	*/
8021	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8022	{
8023	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8024	{
8025	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
8026	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8027	}
8028	return VINF_SUCCESS;
8029	}
8030
8031
8032	/**
8033	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
8034	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
8035	*
8036	* @param pVCpu The cross context virtual CPU structure.
8037	* @param pVmcsInfo The VMCS info. object.
8038	*/
8039	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8040	{
8041	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8042	{
8043	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
8044	{
8045	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8046	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8047	AssertRC(rc);
8048	Log4Func(("Setup NMI-window exiting\n"));
8049	}
8050	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
8051	}
8052
8053
8054	/**
8055	* Clears the NMI-window exiting control in the VMCS.
8056	*
8057	* @param pVmcsInfo The VMCS info. object.
8058	*/
8059	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8060	{
8061	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8062	{
8063	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8064	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8065	}
8066	return VINF_SUCCESS;
8067	}
8068
8069
8070	/**
8071	* Does the necessary state syncing before returning to ring-3 for any reason
8072	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
8073	*
8074	* @returns VBox status code.
8075	* @param pVCpu The cross context virtual CPU structure.
8076	* @param fImportState Whether to import the guest state from the VMCS back
8077	* to the guest-CPU context.
8078	*
8079	* @remarks No-long-jmp zone!!!
8080	*/
8081	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
8082	{
8083	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8084	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8085
8086	RTCPUID idCpu = RTMpCpuId();
8087	Log4Func(("HostCpuId=%u\n", idCpu));
8088
8089	/*
8090	* !!! IMPORTANT !!!
8091	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
8092	*/
8093
8094	/* Save the guest state if necessary. */
8095	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8096	if (fImportState)
8097	{
8098	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8099	AssertRCReturn(rc, rc);
8100	}
8101
8102	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8103	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8104	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8105
8106	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8107	#ifdef VBOX_STRICT
8108	if (CPUMIsHyperDebugStateActive(pVCpu))
8109	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8110	#endif
8111	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8112	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8113	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8114
8115	#if HC_ARCH_BITS == 64
8116	/* Restore host-state bits that VT-x only restores partially. */
8117	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8118	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8119	{
8120	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8121	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8122	}
8123	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8124	#endif
8125
8126	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8127	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8128	{
8129	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8130	if (!fImportState)
8131	{
8132	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8133	AssertRCReturn(rc, rc);
8134	}
8135	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8136	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8137	}
8138	else
8139	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8140
8141	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8142	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8143
8144	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8145	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8146	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8147	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8148	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8149	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8150	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8151	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8152	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8153
8154	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8155
8156	/** @todo This partially defeats the purpose of having preemption hooks.
8157	* The problem is, deregistering the hooks should be moved to a place that
8158	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8159	* context.
8160	*/
8161	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8162	AssertRCReturn(rc, rc);
8163
8164	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8165	NOREF(idCpu);
8166	return VINF_SUCCESS;
8167	}
8168
8169
8170	/**
8171	* Leaves the VT-x session.
8172	*
8173	* @returns VBox status code.
8174	* @param pVCpu The cross context virtual CPU structure.
8175	*
8176	* @remarks No-long-jmp zone!!!
8177	*/
8178	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8179	{
8180	HM_DISABLE_PREEMPT(pVCpu);
8181	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8182	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8183	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8184
8185	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8186	and done this from the VMXR0ThreadCtxCallback(). */
8187	if (!pVCpu->hm.s.fLeaveDone)
8188	{
8189	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8190	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8191	pVCpu->hm.s.fLeaveDone = true;
8192	}
8193	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8194
8195	/*
8196	* !!! IMPORTANT !!!
8197	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8198	*/
8199
8200	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8201	/** @todo Deregistering here means we need to VMCLEAR always
8202	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8203	* for calling VMMR0ThreadCtxHookDisable here! */
8204	VMMR0ThreadCtxHookDisable(pVCpu);
8205
8206	/* Leave HM context. This takes care of local init (term). */
8207	int rc = HMR0LeaveCpu(pVCpu);
8208
8209	HM_RESTORE_PREEMPT();
8210	return rc;
8211	}
8212
8213
8214	/**
8215	* Does the necessary state syncing before doing a longjmp to ring-3.
8216	*
8217	* @returns VBox status code.
8218	* @param pVCpu The cross context virtual CPU structure.
8219	*
8220	* @remarks No-long-jmp zone!!!
8221	*/
8222	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8223	{
8224	return hmR0VmxLeaveSession(pVCpu);
8225	}
8226
8227
8228	/**
8229	* Take necessary actions before going back to ring-3.
8230	*
8231	* An action requires us to go back to ring-3. This function does the necessary
8232	* steps before we can safely return to ring-3. This is not the same as longjmps
8233	* to ring-3, this is voluntary and prepares the guest so it may continue
8234	* executing outside HM (recompiler/IEM).
8235	*
8236	* @returns VBox status code.
8237	* @param pVCpu The cross context virtual CPU structure.
8238	* @param rcExit The reason for exiting to ring-3. Can be
8239	* VINF_VMM_UNKNOWN_RING3_CALL.
8240	*/
8241	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8242	{
8243	Assert(pVCpu);
8244	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8245
8246	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8247	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8248	{
8249	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8250	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8251	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8252	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8253	}
8254
8255	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8256	VMMRZCallRing3Disable(pVCpu);
8257	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8258
8259	/*
8260	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8261	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8262	*
8263	* This is because execution may continue from ring-3 and we would need to inject
8264	* the event from there (hence place it back in TRPM).
8265	*/
8266	if (pVCpu->hm.s.Event.fPending)
8267	{
8268	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8269	Assert(!pVCpu->hm.s.Event.fPending);
8270
8271	/* Clear the events from the VMCS. */
8272	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8273	AssertRCReturn(rc, rc);
8274	}
8275	#ifdef VBOX_STRICT
8276	else
8277	{
8278	/*
8279	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8280	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8281	* occasionally, see @bugref{9180#c42}.
8282	*/
8283	uint32_t uEntryIntInfo;
8284	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8285	AssertRC(rc);
8286	Assert(!VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8287	}
8288	#endif
8289
8290	/*
8291	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8292	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8293	* (e.g. TPR below threshold).
8294	*/
8295	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8296	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8297	AssertRCReturn(rc, rc);
8298
8299	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8300	and if we're injecting an event we should have a TRPM trap pending. */
8301	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8302	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8303	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8304	#endif
8305
8306	/* Save guest state and restore host state bits. */
8307	rc = hmR0VmxLeaveSession(pVCpu);
8308	AssertRCReturn(rc, rc);
8309	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8310
8311	/* Thread-context hooks are unregistered at this point!!! */
8312
8313	/* Sync recompiler state. */
8314	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8315	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8316	\| CPUM_CHANGED_LDTR
8317	\| CPUM_CHANGED_GDTR
8318	\| CPUM_CHANGED_IDTR
8319	\| CPUM_CHANGED_TR
8320	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8321	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8322	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8323	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8324
8325	Assert(!pVCpu->hm.s.fClearTrapFlag);
8326
8327	/* Update the exit-to-ring 3 reason. */
8328	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8329
8330	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8331	if ( rcExit != VINF_EM_RAW_INTERRUPT
8332	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8333	{
8334	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8335	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8336	}
8337
8338	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8339
8340	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
8341	VMMRZCallRing3RemoveNotification(pVCpu);
8342	VMMRZCallRing3Enable(pVCpu);
8343
8344	return rc;
8345	}
8346
8347
8348	/**
8349	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8350	* longjump to ring-3 and possibly get preempted.
8351	*
8352	* @returns VBox status code.
8353	* @param pVCpu The cross context virtual CPU structure.
8354	* @param enmOperation The operation causing the ring-3 longjump.
8355	* @param pvUser User argument, currently unused, NULL.
8356	*/
8357	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
8358	{
8359	RT_NOREF(pvUser);
8360	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8361	{
8362	/*
8363	* !!! IMPORTANT !!!
8364	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8365	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8366	*/
8367	VMMRZCallRing3RemoveNotification(pVCpu);
8368	VMMRZCallRing3Disable(pVCpu);
8369	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
8370	RTThreadPreemptDisable(&PreemptState);
8371
8372	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8373	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8374	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8375	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8376
8377	#if HC_ARCH_BITS == 64
8378	/* Restore host-state bits that VT-x only restores partially. */
8379	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8380	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8381	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8382	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8383	#endif
8384
8385	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8386	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8387	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8388
8389	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8390	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8391	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8392
8393	/* Clear the current VMCS data back to memory. */
8394	hmR0VmxClearVmcs(pVmcsInfo);
8395
8396	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8397	VMMR0ThreadCtxHookDisable(pVCpu);
8398	HMR0LeaveCpu(pVCpu);
8399	RTThreadPreemptRestore(&PreemptState);
8400	return VINF_SUCCESS;
8401	}
8402
8403	Assert(pVCpu);
8404	Assert(pvUser);
8405	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8406	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8407
8408	VMMRZCallRing3Disable(pVCpu);
8409	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8410
8411	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8412
8413	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8414	AssertRCReturn(rc, rc);
8415
8416	VMMRZCallRing3Enable(pVCpu);
8417	return VINF_SUCCESS;
8418	}
8419
8420
8421	/**
8422	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8423	* stack.
8424	*
8425	* @returns Strict VBox status code (i.e. informational status codes too).
8426	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8427	* @param pVCpu The cross context virtual CPU structure.
8428	* @param uValue The value to push to the guest stack.
8429	*/
8430	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
8431	{
8432	/*
8433	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8434	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8435	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8436	*/
8437	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8438	if (pCtx->sp == 1)
8439	return VINF_EM_RESET;
8440	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8441	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8442	AssertRC(rc);
8443	return rc;
8444	}
8445
8446
8447	/**
8448	* Injects an event into the guest upon VM-entry by updating the relevant fields
8449	* in the VM-entry area in the VMCS.
8450	*
8451	* @returns Strict VBox status code (i.e. informational status codes too).
8452	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8453	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8454	*
8455	* @param pVCpu The cross context virtual CPU structure.
8456	* @param pVmxTransient The VMX-transient structure.
8457	* @param pEvent The event being injected.
8458	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state.
8459	* This will be updated if necessary. This cannot not
8460	* be NULL.
8461	* @param fStepping Whether we're single-stepping guest execution and
8462	* should return VINF_EM_DBG_STEPPED if the event is
8463	* injected directly (registers modified by us, not by
8464	* hardware on VM-entry).
8465	*/
8466	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8467	uint32_t *pfIntrState)
8468	{
8469	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8470	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8471	Assert(pfIntrState);
8472
8473	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8474	uint32_t u32IntInfo = pEvent->u64IntInfo;
8475	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8476	uint32_t const cbInstr = pEvent->cbInstr;
8477	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8478	uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8479	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8480
8481	#ifdef VBOX_STRICT
8482	/*
8483	* Validate the error-code-valid bit for hardware exceptions.
8484	* No error codes for exceptions in real-mode.
8485	*
8486	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8487	*/
8488	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8489	&& !CPUMIsGuestInRealModeEx(pCtx))
8490	{
8491	switch (uVector)
8492	{
8493	case X86_XCPT_PF:
8494	case X86_XCPT_DF:
8495	case X86_XCPT_TS:
8496	case X86_XCPT_NP:
8497	case X86_XCPT_SS:
8498	case X86_XCPT_GP:
8499	case X86_XCPT_AC:
8500	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8501	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8502	RT_FALL_THRU();
8503	default:
8504	break;
8505	}
8506	}
8507
8508	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8509	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8510	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8511	#endif
8512
8513	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8514
8515	/*
8516	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8517	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8518	* interrupt handler in the (real-mode) guest.
8519	*
8520	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8521	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8522	*/
8523	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8524	{
8525	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8526	{
8527	/*
8528	* For CPUs with unrestricted guest execution enabled and with the guest
8529	* in real-mode, we must not set the deliver-error-code bit.
8530	*
8531	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8532	*/
8533	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8534	}
8535	else
8536	{
8537	PVM pVM = pVCpu->CTX_SUFF(pVM);
8538	Assert(PDMVmmDevHeapIsEnabled(pVM));
8539	Assert(pVM->hm.s.vmx.pRealModeTSS);
8540	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8541
8542	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8543	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8544	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8545	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8546	AssertRCReturn(rc2, rc2);
8547
8548	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8549	size_t const cbIdtEntry = sizeof(X86IDTR16);
8550	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8551	{
8552	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8553	if (uVector == X86_XCPT_DF)
8554	return VINF_EM_RESET;
8555
8556	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8557	No error codes for exceptions in real-mode. */
8558	if (uVector == X86_XCPT_GP)
8559	{
8560	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8561	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8562	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8563	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8564	HMEVENT EventXcptDf;
8565	RT_ZERO(EventXcptDf);
8566	EventXcptDf.u64IntInfo = uXcptDfInfo;
8567	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8568	}
8569
8570	/*
8571	* If we're injecting an event with no valid IDT entry, inject a #GP.
8572	* No error codes for exceptions in real-mode.
8573	*
8574	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8575	*/
8576	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8577	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8578	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8579	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8580	HMEVENT EventXcptGp;
8581	RT_ZERO(EventXcptGp);
8582	EventXcptGp.u64IntInfo = uXcptGpInfo;
8583	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8584	}
8585
8586	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8587	uint16_t uGuestIp = pCtx->ip;
8588	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8589	{
8590	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8591	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8592	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8593	}
8594	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8595	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8596
8597	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8598	X86IDTR16 IdtEntry;
8599	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8600	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8601	AssertRCReturn(rc2, rc2);
8602
8603	/* Construct the stack frame for the interrupt/exception handler. */
8604	VBOXSTRICTRC rcStrict;
8605	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8606	if (rcStrict == VINF_SUCCESS)
8607	{
8608	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8609	if (rcStrict == VINF_SUCCESS)
8610	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8611	}
8612
8613	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8614	if (rcStrict == VINF_SUCCESS)
8615	{
8616	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8617	pCtx->rip = IdtEntry.offSel;
8618	pCtx->cs.Sel = IdtEntry.uSel;
8619	pCtx->cs.ValidSel = IdtEntry.uSel;
8620	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8621	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8622	&& uVector == X86_XCPT_PF)
8623	pCtx->cr2 = GCPtrFault;
8624
8625	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8626	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8627	\| HM_CHANGED_GUEST_RSP);
8628
8629	/*
8630	* If we delivered a hardware exception (other than an NMI) and if there was
8631	* block-by-STI in effect, we should clear it.
8632	*/
8633	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8634	{
8635	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8636	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8637	Log4Func(("Clearing inhibition due to STI\n"));
8638	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8639	}
8640
8641	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8642	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8643
8644	/*
8645	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8646	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8647	*/
8648	pVCpu->hm.s.Event.fPending = false;
8649
8650	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8651	if (fStepping)
8652	rcStrict = VINF_EM_DBG_STEPPED;
8653	}
8654	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8655	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8656	return rcStrict;
8657	}
8658	}
8659
8660	/*
8661	* Validate.
8662	*/
8663	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8664	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8665
8666	/*
8667	* Inject the event into the VMCS.
8668	*/
8669	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8670	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8671	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8672	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8673	AssertRCReturn(rc, rc);
8674
8675	/*
8676	* Update guest CR2 if this is a page-fault.
8677	*/
8678	if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8679	&& uVector == X86_XCPT_PF)
8680	pCtx->cr2 = GCPtrFault;
8681
8682	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8683	return VINF_SUCCESS;
8684	}
8685
8686
8687	/**
8688	* Evaluates the event to be delivered to the guest and sets it as the pending
8689	* event.
8690	*
8691	* @returns Strict VBox status code (i.e. informational status codes too).
8692	* @param pVCpu The cross context virtual CPU structure.
8693	* @param pVmxTransient The VMX-transient structure.
8694	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8695	*/
8696	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8697	{
8698	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8699	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8700
8701	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
8702	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmcsInfo);
8703	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8704	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8705	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8706
8707	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8708	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8709	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8710	Assert(!TRPMHasTrap(pVCpu));
8711	Assert(pfIntrState);
8712
8713	*pfIntrState = fIntrState;
8714
8715	/*
8716	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
8717	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
8718	*/
8719	/** @todo SMI. SMIs take priority over NMIs. */
8720	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
8721	{
8722	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8723	if ( !pVCpu->hm.s.Event.fPending
8724	&& !fBlockNmi
8725	&& !fBlockSti
8726	&& !fBlockMovSS)
8727	{
8728	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8729	if ( pVmxTransient->fIsNestedGuest
8730	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8731	return IEMExecVmxVmexitNmi(pVCpu);
8732	#endif
8733	hmR0VmxSetPendingXcptNmi(pVCpu);
8734	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8735	Log4Func(("Pending NMI\n"));
8736	}
8737	else
8738	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8739	}
8740	/*
8741	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
8742	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
8743	*/
8744	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8745	&& !pVCpu->hm.s.fSingleInstruction)
8746	{
8747	Assert(!DBGFIsStepping(pVCpu));
8748	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8749	AssertRCReturn(rc, rc);
8750	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8751	if ( !pVCpu->hm.s.Event.fPending
8752	&& !fBlockInt
8753	&& !fBlockSti
8754	&& !fBlockMovSS)
8755	{
8756	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8757	if ( pVmxTransient->fIsNestedGuest
8758	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
8759	{
8760	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector /, true / fIntPending */);
8761	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8762	return rcStrict;
8763	}
8764	#endif
8765	uint8_t u8Interrupt;
8766	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8767	if (RT_SUCCESS(rc))
8768	{
8769	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8770	if ( pVmxTransient->fIsNestedGuest
8771	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8772	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8773	{
8774	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8775	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8776	return rcStrict;
8777	}
8778	#endif
8779	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8780	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
8781	}
8782	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8783	{
8784	if ( !pVmxTransient->fIsNestedGuest
8785	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8786	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
8787	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8788
8789	/*
8790	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8791	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8792	* need to re-set this force-flag here.
8793	*/
8794	}
8795	else
8796	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8797	}
8798	else
8799	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8800	}
8801
8802	return VINF_SUCCESS;
8803	}
8804
8805
8806	/**
8807	* Injects any pending events into the guest if the guest is in a state to
8808	* receive them.
8809	*
8810	* @returns Strict VBox status code (i.e. informational status codes too).
8811	* @param pVCpu The cross context virtual CPU structure.
8812	* @param pVmxTransient The VMX-transient structure.
8813	* @param fIntrState The VT-x guest-interruptibility state.
8814	* @param fStepping Whether we are single-stepping the guest using the
8815	* hypervisor debugger and should return
8816	* VINF_EM_DBG_STEPPED if the event was dispatched
8817	* directly.
8818	*/
8819	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
8820	{
8821	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8822	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8823
8824	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8825	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8826
8827	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8828	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8829	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8830	Assert(!TRPMHasTrap(pVCpu));
8831
8832	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8833	if (pVCpu->hm.s.Event.fPending)
8834	{
8835	/*
8836	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8837	* pending even while injecting an event and in this case, we want a VM-exit as soon as
8838	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
8839	*
8840	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8841	*/
8842	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8843	#ifdef VBOX_STRICT
8844	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8845	{
8846	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
8847	Assert(!fBlockInt);
8848	Assert(!fBlockSti);
8849	Assert(!fBlockMovSS);
8850	}
8851	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
8852	{
8853	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8854	Assert(!fBlockSti);
8855	Assert(!fBlockMovSS);
8856	Assert(!fBlockNmi);
8857	}
8858	#endif
8859	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8860	uIntType));
8861
8862	/*
8863	* Inject the event and get any changes to the guest-interruptibility state.
8864	*
8865	* The guest-interruptibility state may need to be updated if we inject the event
8866	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
8867	*/
8868	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
8869	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8870
8871	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8872	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8873	else
8874	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8875	}
8876
8877	/*
8878	* Update the guest-interruptibility state.
8879	*
8880	* This is required for the real-on-v86 software interrupt injection case above, as well as
8881	* updates to the guest state from ring-3 or IEM/REM.
8882	*/
8883	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
8884	AssertRCReturn(rc, rc);
8885
8886	/*
8887	* There's no need to clear the VM-entry interruption-information field here if we're not
8888	* injecting anything. VT-x clears the valid bit on every VM-exit.
8889	*
8890	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8891	*/
8892
8893	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8894	NOREF(fBlockMovSS); NOREF(fBlockSti);
8895	return rcStrict;
8896	}
8897
8898
8899	/**
8900	* Enters the VT-x session.
8901	*
8902	* @returns VBox status code.
8903	* @param pVCpu The cross context virtual CPU structure.
8904	*/
8905	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
8906	{
8907	AssertPtr(pVCpu);
8908	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8909	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8910
8911	LogFlowFunc(("pVCpu=%p\n", pVCpu));
8912	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8913	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8914
8915	#ifdef VBOX_STRICT
8916	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8917	RTCCUINTREG uHostCR4 = ASMGetCR4();
8918	if (!(uHostCR4 & X86_CR4_VMXE))
8919	{
8920	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8921	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8922	}
8923	#endif
8924
8925	/*
8926	* Load the appropriate VMCS as the current and active one.
8927	*/
8928	PVMXVMCSINFO pVmcsInfo;
8929	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
8930	if (!fInNestedGuestMode)
8931	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
8932	else
8933	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
8934	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
8935	if (RT_SUCCESS(rc))
8936	{
8937	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
8938	pVCpu->hm.s.fLeaveDone = false;
8939	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8940
8941	/*
8942	* Do the EMT scheduled L1D flush here if needed.
8943	*/
8944	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8945	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8946	}
8947	return rc;
8948	}
8949
8950
8951	/**
8952	* The thread-context callback (only on platforms which support it).
8953	*
8954	* @param enmEvent The thread-context event.
8955	* @param pVCpu The cross context virtual CPU structure.
8956	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8957	* @thread EMT(pVCpu)
8958	*/
8959	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8960	{
8961	NOREF(fGlobalInit);
8962
8963	switch (enmEvent)
8964	{
8965	case RTTHREADCTXEVENT_OUT:
8966	{
8967	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8968	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8969	VMCPU_ASSERT_EMT(pVCpu);
8970
8971	/* No longjmps (logger flushes, locks) in this fragile context. */
8972	VMMRZCallRing3Disable(pVCpu);
8973	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8974
8975	/* Restore host-state (FPU, debug etc.) */
8976	if (!pVCpu->hm.s.fLeaveDone)
8977	{
8978	/*
8979	* Do -not- import the guest-state here as we might already be in the middle of importing
8980	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8981	*/
8982	hmR0VmxLeave(pVCpu, false /* fImportState */);
8983	pVCpu->hm.s.fLeaveDone = true;
8984	}
8985
8986	/* Leave HM context, takes care of local init (term). */
8987	int rc = HMR0LeaveCpu(pVCpu);
8988	AssertRC(rc);
8989
8990	/* Restore longjmp state. */
8991	VMMRZCallRing3Enable(pVCpu);
8992	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8993	break;
8994	}
8995
8996	case RTTHREADCTXEVENT_IN:
8997	{
8998	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8999	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9000	VMCPU_ASSERT_EMT(pVCpu);
9001
9002	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
9003	VMMRZCallRing3Disable(pVCpu);
9004	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
9005
9006	/* Initialize the bare minimum state required for HM. This takes care of
9007	initializing VT-x if necessary (onlined CPUs, local init etc.) */
9008	int rc = hmR0EnterCpu(pVCpu);
9009	AssertRC(rc);
9010	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9011	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9012
9013	/* Load the active VMCS as the current one. */
9014	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9015	rc = hmR0VmxLoadVmcs(pVmcsInfo);
9016	AssertRC(rc);
9017	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9018	pVCpu->hm.s.fLeaveDone = false;
9019
9020	/* Do the EMT scheduled L1D flush if needed. */
9021	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9022	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9023
9024	/* Restore longjmp state. */
9025	VMMRZCallRing3Enable(pVCpu);
9026	break;
9027	}
9028
9029	default:
9030	break;
9031	}
9032	}
9033
9034
9035	/**
9036	* Exports the host state into the VMCS host-state area.
9037	* Sets up the VM-exit MSR-load area.
9038	*
9039	* The CPU state will be loaded from these fields on every successful VM-exit.
9040	*
9041	* @returns VBox status code.
9042	* @param pVCpu The cross context virtual CPU structure.
9043	*
9044	* @remarks No-long-jump zone!!!
9045	*/
9046	static int hmR0VmxExportHostState(PVMCPU pVCpu)
9047	{
9048	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9049
9050	int rc = VINF_SUCCESS;
9051	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
9052	{
9053	rc = hmR0VmxExportHostControlRegs();
9054	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9055
9056	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
9057	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9058
9059	rc = hmR0VmxExportHostMsrs(pVCpu);
9060	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9061
9062	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
9063	}
9064	return rc;
9065	}
9066
9067
9068	/**
9069	* Saves the host state in the VMCS host-state.
9070	*
9071	* @returns VBox status code.
9072	* @param pVCpu The cross context virtual CPU structure.
9073	*
9074	* @remarks No-long-jump zone!!!
9075	*/
9076	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
9077	{
9078	AssertPtr(pVCpu);
9079	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9080
9081	/*
9082	* Export the host state here while entering HM context.
9083	* When thread-context hooks are used, we might get preempted and have to re-save the host
9084	* state but most of the time we won't be, so do it here before we disable interrupts.
9085	*/
9086	return hmR0VmxExportHostState(pVCpu);
9087	}
9088
9089
9090	/**
9091	* Exports the guest state into the VMCS guest-state area.
9092	*
9093	* The will typically be done before VM-entry when the guest-CPU state and the
9094	* VMCS state may potentially be out of sync.
9095	*
9096	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9097	* VM-entry controls.
9098	* Sets up the appropriate VMX non-root function to execute guest code based on
9099	* the guest CPU mode.
9100	*
9101	* @returns VBox strict status code.
9102	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9103	* without unrestricted guest execution and the VMMDev is not presently
9104	* mapped (e.g. EFI32).
9105	*
9106	* @param pVCpu The cross context virtual CPU structure.
9107	* @param pVmxTransient The VMX-transient structure.
9108	*
9109	* @remarks No-long-jump zone!!!
9110	*/
9111	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9112	{
9113	AssertPtr(pVCpu);
9114	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9115	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9116
9117	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9118
9119	/*
9120	* Determine real-on-v86 mode.
9121	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9122	*/
9123	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9124	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9125	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9126	pVmcsInfo->RealMode. fRealOnV86Active = false;
9127	else
9128	{
9129	Assert(!pVmxTransient->fIsNestedGuest);
9130	pVmcsInfo->RealMode.fRealOnV86Active = true;
9131	}
9132
9133	/*
9134	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9135	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9136	*/
9137	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9138	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9139	* be a need to evaluate this everytime since I'm pretty sure we intercept
9140	* all guest paging mode changes. */
9141	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9142	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9143
9144	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9145	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9146
9147	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9148	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9149
9150	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9151	if (rcStrict == VINF_SUCCESS)
9152	{ /* likely */ }
9153	else
9154	{
9155	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9156	return rcStrict;
9157	}
9158
9159	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9160	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9161
9162	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9163	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9164
9165	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9166	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9167
9168	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9169	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9170
9171	rc = hmR0VmxExportGuestRip(pVCpu);
9172	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9173	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9174	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9175
9176	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9177	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9178	\| HM_CHANGED_GUEST_CR2
9179	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9180	\| HM_CHANGED_GUEST_X87
9181	\| HM_CHANGED_GUEST_SSE_AVX
9182	\| HM_CHANGED_GUEST_OTHER_XSAVE
9183	\| HM_CHANGED_GUEST_XCRx
9184	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9185	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9186	\| HM_CHANGED_GUEST_TSC_AUX
9187	\| HM_CHANGED_GUEST_OTHER_MSRS
9188	\| HM_CHANGED_GUEST_HWVIRT /* More accurate PLE handling someday? */
9189	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9190
9191	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9192	return rc;
9193	}
9194
9195
9196	/**
9197	* Exports the state shared between the host and guest into the VMCS.
9198	*
9199	* @param pVCpu The cross context virtual CPU structure.
9200	* @param pVmxTransient The VMX-transient structure.
9201	*
9202	* @remarks No-long-jump zone!!!
9203	*/
9204	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9205	{
9206	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9207	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9208
9209	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9210	{
9211	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9212	AssertRC(rc);
9213	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9214
9215	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9216	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9217	{
9218	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9219	AssertRC(rc);
9220	}
9221	}
9222
9223	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9224	{
9225	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9226	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9227	}
9228
9229	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9230	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9231	}
9232
9233
9234	/**
9235	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9236	*
9237	* @returns Strict VBox status code (i.e. informational status codes too).
9238	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9239	* without unrestricted guest execution and the VMMDev is not presently
9240	* mapped (e.g. EFI32).
9241	*
9242	* @param pVCpu The cross context virtual CPU structure.
9243	* @param pVmxTransient The VMX-transient structure.
9244	*
9245	* @remarks No-long-jump zone!!!
9246	*/
9247	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9248	{
9249	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9250	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9251	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9252
9253	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9254	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9255	#endif
9256
9257	/*
9258	* For many exits it's only RIP that changes and hence try to export it first
9259	* without going through a lot of change flag checks.
9260	*/
9261	VBOXSTRICTRC rcStrict;
9262	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9263	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9264	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9265	{
9266	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9267	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9268	{ /* likely */}
9269	else
9270	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9271	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9272	}
9273	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9274	{
9275	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9276	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9277	{ /* likely */}
9278	else
9279	{
9280	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9281	VBOXSTRICTRC_VAL(rcStrict)));
9282	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9283	return rcStrict;
9284	}
9285	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9286	}
9287	else
9288	rcStrict = VINF_SUCCESS;
9289
9290	#ifdef VBOX_STRICT
9291	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9292	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9293	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9294	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9295	("fCtxChanged=%#RX64\n", fCtxChanged));
9296	#endif
9297	return rcStrict;
9298	}
9299
9300
9301	/**
9302	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9303	* and update error record fields accordingly.
9304	*
9305	* @return VMX_IGS_* return codes.
9306	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9307	* wrong with the guest state.
9308	*
9309	* @param pVCpu The cross context virtual CPU structure.
9310	* @param pVmcsInfo The VMCS info. object.
9311	*
9312	* @remarks This function assumes our cache of the VMCS controls
9313	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9314	*/
9315	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
9316	{
9317	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9318	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9319	uError = (err); \
9320	break; \
9321	} else do { } while (0)
9322
9323	int rc;
9324	PVM pVM = pVCpu->CTX_SUFF(pVM);
9325	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9326	uint32_t uError = VMX_IGS_ERROR;
9327	uint32_t u32Val;
9328	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9329
9330	do
9331	{
9332	/*
9333	* CR0.
9334	*/
9335	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9336	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9337	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
9338	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9339	if (fUnrestrictedGuest)
9340	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
9341
9342	uint32_t u32GuestCr0;
9343	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
9344	AssertRCBreak(rc);
9345	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9346	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9347	if ( !fUnrestrictedGuest
9348	&& (u32GuestCr0 & X86_CR0_PG)
9349	&& !(u32GuestCr0 & X86_CR0_PE))
9350	{
9351	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9352	}
9353
9354	/*
9355	* CR4.
9356	*/
9357	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9358	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9359
9360	uint32_t u32GuestCr4;
9361	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
9362	AssertRCBreak(rc);
9363	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9364	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9365
9366	/*
9367	* IA32_DEBUGCTL MSR.
9368	*/
9369	uint64_t u64Val;
9370	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9371	AssertRCBreak(rc);
9372	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9373	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9374	{
9375	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9376	}
9377	uint64_t u64DebugCtlMsr = u64Val;
9378
9379	#ifdef VBOX_STRICT
9380	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9381	AssertRCBreak(rc);
9382	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9383	#endif
9384	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9385
9386	/*
9387	* RIP and RFLAGS.
9388	*/
9389	uint32_t u32Eflags;
9390	#if HC_ARCH_BITS == 64
9391	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9392	AssertRCBreak(rc);
9393	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9394	if ( !fLongModeGuest
9395	\|\| !pCtx->cs.Attr.n.u1Long)
9396	{
9397	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9398	}
9399	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9400	* must be identical if the "IA-32e mode guest" VM-entry
9401	* control is 1 and CS.L is 1. No check applies if the
9402	* CPU supports 64 linear-address bits. */
9403
9404	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9405	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9406	AssertRCBreak(rc);
9407	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9408	VMX_IGS_RFLAGS_RESERVED);
9409	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9410	u32Eflags = u64Val;
9411	#else
9412	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9413	AssertRCBreak(rc);
9414	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9415	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9416	#endif
9417
9418	if ( fLongModeGuest
9419	\|\| ( fUnrestrictedGuest
9420	&& !(u32GuestCr0 & X86_CR0_PE)))
9421	{
9422	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9423	}
9424
9425	uint32_t u32EntryInfo;
9426	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9427	AssertRCBreak(rc);
9428	if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9429	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9430	{
9431	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9432	}
9433
9434	/*
9435	* 64-bit checks.
9436	*/
9437	#if HC_ARCH_BITS == 64
9438	if (fLongModeGuest)
9439	{
9440	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9441	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9442	}
9443
9444	if ( !fLongModeGuest
9445	&& (u32GuestCr4 & X86_CR4_PCIDE))
9446	{
9447	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9448	}
9449
9450	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9451	* 51:32 beyond the processor's physical-address width are 0. */
9452
9453	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9454	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9455	{
9456	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9457	}
9458
9459	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9460	AssertRCBreak(rc);
9461	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9462
9463	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9464	AssertRCBreak(rc);
9465	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9466	#endif
9467
9468	/*
9469	* PERF_GLOBAL MSR.
9470	*/
9471	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9472	{
9473	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9474	AssertRCBreak(rc);
9475	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9476	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9477	}
9478
9479	/*
9480	* PAT MSR.
9481	*/
9482	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9483	{
9484	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9485	AssertRCBreak(rc);
9486	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9487	for (unsigned i = 0; i < 8; i++)
9488	{
9489	uint8_t u8Val = (u64Val & 0xff);
9490	if ( u8Val != 0 /* UC */
9491	&& u8Val != 1 /* WC */
9492	&& u8Val != 4 /* WT */
9493	&& u8Val != 5 /* WP */
9494	&& u8Val != 6 /* WB */
9495	&& u8Val != 7 /* UC- */)
9496	{
9497	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9498	}
9499	u64Val >>= 8;
9500	}
9501	}
9502
9503	/*
9504	* EFER MSR.
9505	*/
9506	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9507	{
9508	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9509	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9510	AssertRCBreak(rc);
9511	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9512	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9513	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9514	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9515	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9516	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9517	* iemVmxVmentryCheckGuestState(). */
9518	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9519	\|\| !(u32GuestCr0 & X86_CR0_PG)
9520	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9521	VMX_IGS_EFER_LMA_LME_MISMATCH);
9522	}
9523
9524	/*
9525	* Segment registers.
9526	*/
9527	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9528	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9529	if (!(u32Eflags & X86_EFL_VM))
9530	{
9531	/* CS */
9532	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9533	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9534	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9535	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9536	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9537	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9538	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9539	/* CS cannot be loaded with NULL in protected mode. */
9540	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9541	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9542	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9543	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9544	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9545	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9546	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9547	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9548	else
9549	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9550
9551	/* SS */
9552	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9553	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9554	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9555	if ( !(pCtx->cr0 & X86_CR0_PE)
9556	\|\| pCtx->cs.Attr.n.u4Type == 3)
9557	{
9558	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9559	}
9560	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9561	{
9562	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9563	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9564	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9565	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9566	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9567	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9568	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9569	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9570	}
9571
9572	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9573	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9574	{
9575	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9576	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9577	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9578	\|\| pCtx->ds.Attr.n.u4Type > 11
9579	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9580	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9581	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9582	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9583	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9584	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9585	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9586	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9587	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9588	}
9589	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9590	{
9591	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9592	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9593	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9594	\|\| pCtx->es.Attr.n.u4Type > 11
9595	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9596	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9597	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9598	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9599	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9600	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9601	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9602	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9603	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9604	}
9605	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9606	{
9607	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9608	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9609	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9610	\|\| pCtx->fs.Attr.n.u4Type > 11
9611	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9612	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9613	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9614	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9615	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9616	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9617	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9618	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9619	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9620	}
9621	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9622	{
9623	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9624	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9625	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9626	\|\| pCtx->gs.Attr.n.u4Type > 11
9627	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9628	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9629	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9630	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9631	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9632	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9633	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9634	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9635	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9636	}
9637	/* 64-bit capable CPUs. */
9638	#if HC_ARCH_BITS == 64
9639	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9640	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9641	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9642	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9643	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9644	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9645	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9646	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9647	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9648	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9649	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9650	#endif
9651	}
9652	else
9653	{
9654	/* V86 mode checks. */
9655	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9656	if (pVmcsInfo->RealMode.fRealOnV86Active)
9657	{
9658	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9659	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9660	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9661	}
9662	else
9663	{
9664	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9665	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9666	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9667	}
9668
9669	/* CS */
9670	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9671	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9672	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9673	/* SS */
9674	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9675	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9676	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9677	/* DS */
9678	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9679	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9680	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9681	/* ES */
9682	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9683	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9684	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9685	/* FS */
9686	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9687	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9688	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9689	/* GS */
9690	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9691	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9692	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9693	/* 64-bit capable CPUs. */
9694	#if HC_ARCH_BITS == 64
9695	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9696	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9697	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9698	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9699	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9700	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9701	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9702	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9703	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9704	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9705	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9706	#endif
9707	}
9708
9709	/*
9710	* TR.
9711	*/
9712	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9713	/* 64-bit capable CPUs. */
9714	#if HC_ARCH_BITS == 64
9715	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9716	#endif
9717	if (fLongModeGuest)
9718	{
9719	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9720	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9721	}
9722	else
9723	{
9724	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9725	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9726	VMX_IGS_TR_ATTR_TYPE_INVALID);
9727	}
9728	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9729	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9730	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9731	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9732	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9733	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9734	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9735	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9736
9737	/*
9738	* GDTR and IDTR.
9739	*/
9740	#if HC_ARCH_BITS == 64
9741	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9742	AssertRCBreak(rc);
9743	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9744
9745	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9746	AssertRCBreak(rc);
9747	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9748	#endif
9749
9750	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9751	AssertRCBreak(rc);
9752	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9753
9754	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9755	AssertRCBreak(rc);
9756	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9757
9758	/*
9759	* Guest Non-Register State.
9760	*/
9761	/* Activity State. */
9762	uint32_t u32ActivityState;
9763	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9764	AssertRCBreak(rc);
9765	HMVMX_CHECK_BREAK( !u32ActivityState
9766	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9767	VMX_IGS_ACTIVITY_STATE_INVALID);
9768	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9769	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9770	uint32_t u32IntrState;
9771	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9772	AssertRCBreak(rc);
9773	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9774	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9775	{
9776	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9777	}
9778
9779	/** @todo Activity state and injecting interrupts. Left as a todo since we
9780	* currently don't use activity states but ACTIVE. */
9781
9782	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9783	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9784
9785	/* Guest interruptibility-state. */
9786	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9787	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9788	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9789	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9790	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9791	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9792	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9793	if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
9794	{
9795	if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9796	{
9797	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9798	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9799	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9800	}
9801	else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9802	{
9803	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9804	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9805	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9806	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9807	}
9808	}
9809	/** @todo Assumes the processor is not in SMM. */
9810	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9811	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9812	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9813	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9814	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9815	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9816	&& VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9817	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9818	{
9819	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
9820	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9821	}
9822
9823	/* Pending debug exceptions. */
9824	#if HC_ARCH_BITS == 64
9825	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
9826	AssertRCBreak(rc);
9827	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9828	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9829	u32Val = u64Val; /* For pending debug exceptions checks below. */
9830	#else
9831	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
9832	AssertRCBreak(rc);
9833	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9834	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9835	#endif
9836
9837	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9838	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
9839	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9840	{
9841	if ( (u32Eflags & X86_EFL_TF)
9842	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9843	{
9844	/* Bit 14 is PendingDebug.BS. */
9845	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9846	}
9847	if ( !(u32Eflags & X86_EFL_TF)
9848	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9849	{
9850	/* Bit 14 is PendingDebug.BS. */
9851	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9852	}
9853	}
9854
9855	/* VMCS link pointer. */
9856	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9857	AssertRCBreak(rc);
9858	if (u64Val != UINT64_C(0xffffffffffffffff))
9859	{
9860	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9861	/** @todo Bits beyond the processor's physical-address width MBZ. */
9862	/** @todo 32-bit located in memory referenced by value of this field (as a
9863	* physical address) must contain the processor's VMCS revision ID. */
9864	/** @todo SMM checks. */
9865	}
9866
9867	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9868	* not using nested paging? */
9869	if ( pVM->hm.s.fNestedPaging
9870	&& !fLongModeGuest
9871	&& CPUMIsGuestInPAEModeEx(pCtx))
9872	{
9873	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9874	AssertRCBreak(rc);
9875	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9876
9877	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9878	AssertRCBreak(rc);
9879	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9880
9881	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9882	AssertRCBreak(rc);
9883	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9884
9885	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9886	AssertRCBreak(rc);
9887	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9888	}
9889
9890	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9891	if (uError == VMX_IGS_ERROR)
9892	uError = VMX_IGS_REASON_NOT_FOUND;
9893	} while (0);
9894
9895	pVCpu->hm.s.u32HMError = uError;
9896	return uError;
9897
9898	#undef HMVMX_ERROR_BREAK
9899	#undef HMVMX_CHECK_BREAK
9900	}
9901
9902
9903	/**
9904	* Setup the APIC-access page for virtualizing APIC access.
9905	*
9906	* This can cause a longjumps to R3 due to the acquisition of the PGM lock, hence
9907	* this not done as part of exporting guest state, see @bugref{8721}.
9908	*
9909	* @returns VBox status code.
9910	* @param pVCpu The cross context virtual CPU structure.
9911	*/
9912	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
9913	{
9914	PVM pVM = pVCpu->CTX_SUFF(pVM);
9915	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
9916
9917	Assert(PDMHasApic(pVM));
9918	Assert(u64MsrApicBase);
9919
9920	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
9921	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
9922
9923	/* Unalias any existing mapping. */
9924	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
9925	AssertRCReturn(rc, rc);
9926
9927	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
9928	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
9929	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
9930	AssertRCReturn(rc, rc);
9931
9932	/* Update the per-VCPU cache of the APIC base MSR. */
9933	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
9934	return VINF_SUCCESS;
9935	}
9936
9937
9938	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9939	/**
9940	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
9941	* nested-guest using hardware-assisted VMX.
9942	*
9943	* @param pVCpu The cross context virtual CPU structure.
9944	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
9945	* @param pVmcsInfoGst The guest VMCS info. object.
9946	*/
9947	static void hmR0VmxMergeMsrBitmapNested(PCVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
9948	{
9949	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
9950	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
9951	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
9952	Assert(pu64MsrBitmapNstGst);
9953	Assert(pu64MsrBitmapGst);
9954	Assert(pu64MsrBitmap);
9955
9956	/*
9957	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
9958	* MSR that is intercepted by the guest is also intercepted while executing the
9959	* nested-guest using hardware-assisted VMX.
9960	*/
9961	uint32_t const cbFrag = sizeof(uint64_t);
9962	uint32_t const cFrags = X86_PAGE_4K_SIZE / cbFrag;
9963	for (uint32_t i = 0; i <= cFrags; i++)
9964	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
9965	}
9966
9967
9968	/**
9969	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
9970	* hardware-assisted VMX execution of the nested-guest.
9971	*
9972	* For a guest, we don't modify these controls once we set up the VMCS.
9973	*
9974	* For nested-guests since the guest hypervisor provides these controls on every
9975	* nested-guest VM-entry and could potentially change them everytime we need to
9976	* merge them before every nested-guest VM-entry.
9977	*
9978	* @returns VBox status code.
9979	* @param pVCpu The cross context virtual CPU structure.
9980	*/
9981	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
9982	{
9983	PVM pVM = pVCpu->CTX_SUFF(pVM);
9984	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
9985	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
9986	Assert(pVmcsNstGst);
9987
9988	/*
9989	* Merge the controls with the requirements of the guest VMCS.
9990	*
9991	* We do not need to validate the nested-guest VMX features specified in the
9992	* nested-guest VMCS with the features supported by the physical CPU as it's
9993	* already done by the VMLAUNCH/VMRESUME instruction emulation.
9994	*
9995	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the
9996	* guest are derived from the VMX features supported by the physical CPU.
9997	*/
9998
9999	/* Pin-based VM-execution controls. */
10000	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
10001
10002	/* Processor-based VM-execution controls. */
10003	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
10004	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
10005	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
10006	\| VMX_PROC_CTLS_USE_TPR_SHADOW
10007	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
10008
10009	/* Secondary processor-based VM-execution controls. */
10010	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
10011	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
10012	\| VMX_PROC_CTLS2_INVPCID
10013	\| VMX_PROC_CTLS2_RDTSCP
10014	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
10015	\| VMX_PROC_CTLS2_APIC_REG_VIRT
10016	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
10017	\| VMX_PROC_CTLS2_VMFUNC));
10018
10019	/*
10020	* VM-entry controls:
10021	* These controls contains state that depends on the nested-guest state (primarily
10022	* EFER MSR) and is thus not constant through VMLAUNCH/VMRESUME and the nested-guest
10023	* VM-exit. Although the nested-hypervisor cannot change it, we need to in order to
10024	* properly continue executing the nested-guest if the EFER MSR changes but does not
10025	* cause a nested-guest VM-exits.
10026	*
10027	* VM-exit controls:
10028	* These controls specify the host state on return. We cannot use the controls from
10029	* the nested-hypervisor state as is as it would contain the guest state rather than
10030	* the host state. Since the host state is subject to change (e.g. preemption, trips
10031	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
10032	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
10033	*
10034	* VM-entry MSR-load:
10035	* The guest MSRs from the VM-entry MSR-load area are already loaded into the
10036	* guest-CPU context by the VMLAUNCH/VMRESUME instruction emulation.
10037	*
10038	* VM-exit MSR-store:
10039	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU
10040	* context back into the VM-exit MSR-store area.
10041	*
10042	* VM-exit MSR-load areas:
10043	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence,
10044	* we can entirely ignore what the nested-hypervisor wants to load here.
10045	*/
10046
10047	/*
10048	* Exception bitmap.
10049	*
10050	* We could remove #UD from the guest bitmap and merge it with the nested-guest
10051	* bitmap here (and avoid doing anything while exporting nested-guest state), but to
10052	* keep the code more flexible if intercepting exceptions become more dynamic in
10053	* the future we do it as part of exporting the nested-guest state.
10054	*/
10055	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
10056
10057	/*
10058	* CR0/CR4 guest/host mask.
10059	*
10060	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest
10061	* must cause VM-exits, so we need to merge them here.
10062	*/
10063	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
10064	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
10065
10066	/*
10067	* Page-fault error-code mask and match.
10068	*
10069	* Although we require unrestricted guest execution (and thereby nested-paging) for
10070	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
10071	* normally intercept #PFs, it might intercept them for debugging purposes.
10072	*
10073	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF
10074	* filters. If the outer guest is intercepting #PFs we must intercept all #PFs.
10075	*/
10076	uint32_t u32XcptPFMask;
10077	uint32_t u32XcptPFMatch;
10078	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10079	{
10080	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10081	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10082	}
10083	else
10084	{
10085	u32XcptPFMask = 0;
10086	u32XcptPFMatch = 0;
10087	}
10088
10089	/*
10090	* Pause-Loop exiting.
10091	*/
10092	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10093	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10094
10095	/*
10096	* I/O Bitmap.
10097	*
10098	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we
10099	* always intercept all I/O port accesses.
10100	*/
10101	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10102
10103	/*
10104	* APIC-access page.
10105	*
10106	* The APIC-access page address has already been initialized while setting up the
10107	* nested-guest VMCS. In theory, even if the guest-physical address is invalid, it
10108	* should not be on any consequence to the host or to the guest for that matter, but
10109	* we only accept valid addresses verified by the VMLAUNCH/VMRESUME instruction
10110	* emulation to keep it simple.
10111	*/
10112
10113	/*
10114	* Virtual-APIC page and TPR threshold.
10115	*
10116	* We shall use the host-physical address of the virtual-APIC page in guest memory directly.
10117	* For this reason, we can access the virtual-APIC page of the nested-guest only using
10118	* PGM physical handlers as we must not assume a kernel virtual-address mapping exists and
10119	* requesting PGM for a mapping could be expensive/resource intensive (PGM mapping cache).
10120	*/
10121	RTHCPHYS HCPhysVirtApic = NIL_RTHCPHYS;
10122	uint32_t const u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10123	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10124	{
10125	int rc = PGMPhysGCPhys2HCPhys(pVM, pVmcsNstGst->u64AddrVirtApic.u, &HCPhysVirtApic);
10126
10127	/*
10128	* If the guest hypervisor has loaded crap into the virtual-APIC page field
10129	* we would fail to obtain a valid host-physical address for its guest-physical
10130	* address.
10131	*
10132	* We currently do not support this scenario. Maybe in the future if there is a
10133	* pressing need we can explore making this particular set of conditions work.
10134	* Right now we just cause a VM-entry failure.
10135	*
10136	* This has already been checked by VMLAUNCH/VMRESUME instruction emulation,
10137	* so should not really failure at the moment.
10138	*/
10139	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10140	}
10141	else
10142	{
10143	/*
10144	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10145	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10146	* be taken care of by EPT/shadow paging.
10147	*/
10148	if (pVM->hm.s.fAllow64BitGuests)
10149	{
10150	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10151	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10152	}
10153	}
10154
10155	/*
10156	* Validate basic assumptions.
10157	*/
10158	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10159	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10160	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10161	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10162
10163	/*
10164	* Commit it to the nested-guest VMCS.
10165	*/
10166	int rc = VINF_SUCCESS;
10167	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10168	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10169	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10170	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10171	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10172	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10173	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10174	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10175	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10176	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10177	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10178	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10179	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10180	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10181	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10182	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10183	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10184	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10185	{
10186	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10187	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10188	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10189	}
10190	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10191	{
10192	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10193	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10194	}
10195	AssertRCReturn(rc, rc);
10196
10197	/*
10198	* Update the nested-guest VMCS cache.
10199	*/
10200	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10201	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10202	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10203	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10204	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10205	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10206	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10207	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10208	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10209
10210	/*
10211	* MSR bitmap.
10212	*
10213	* The MSR bitmap address has already been initialized while setting up the
10214	* nested-guest VMCS, here we need to merge the MSR bitmaps.
10215	*/
10216	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10217	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10218
10219	return VINF_SUCCESS;
10220	}
10221	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10222
10223
10224	/**
10225	* Does the preparations before executing guest code in VT-x.
10226	*
10227	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10228	* recompiler/IEM. We must be cautious what we do here regarding committing
10229	* guest-state information into the VMCS assuming we assuredly execute the
10230	* guest in VT-x mode.
10231	*
10232	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10233	* the common-state (TRPM/forceflags), we must undo those changes so that the
10234	* recompiler/IEM can (and should) use them when it resumes guest execution.
10235	* Otherwise such operations must be done when we can no longer exit to ring-3.
10236	*
10237	* @returns Strict VBox status code (i.e. informational status codes too).
10238	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10239	* have been disabled.
10240	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10241	* double-fault into the guest.
10242	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10243	* dispatched directly.
10244	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10245	*
10246	* @param pVCpu The cross context virtual CPU structure.
10247	* @param pVmxTransient The VMX-transient structure.
10248	* @param fStepping Whether we are single-stepping the guest in the
10249	* hypervisor debugger. Makes us ignore some of the reasons
10250	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10251	* if event dispatching took place.
10252	*/
10253	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10254	{
10255	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10256
10257	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10258	if (pVmxTransient->fIsNestedGuest)
10259	{
10260	RT_NOREF2(pVCpu, fStepping);
10261	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10262	return VINF_EM_RESCHEDULE_REM;
10263	}
10264	#endif
10265
10266	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10267	PGMRZDynMapFlushAutoSet(pVCpu);
10268	#endif
10269
10270	/*
10271	* Check and process force flag actions, some of which might require us to go back to ring-3.
10272	*/
10273	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10274	if (rcStrict == VINF_SUCCESS)
10275	{ /* FFs don't get set all the time. */ }
10276	else
10277	return rcStrict;
10278
10279	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10280	/*
10281	* Switch to the nested-guest VMCS as we may have transitioned into executing
10282	* the nested-guest without leaving ring-0. Otherwise, if we came from ring-3
10283	* we would load the nested-guest VMCS while entering the VMX ring-0 session.
10284	*
10285	* We do this as late as possible to minimize (though not completely remove)
10286	* clearing/loading VMCS again due to premature trips to ring-3 above.
10287	*/
10288	if (pVmxTransient->fIsNestedGuest)
10289	{
10290	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10291	{
10292	/*
10293	* Ensure we have synced everything from the guest VMCS and also flag that
10294	* that we need to export the full (nested) guest-CPU context to the
10295	* nested-guest VMCS.
10296	*/
10297	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
10298	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST);
10299
10300	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
10301	int rc = hmR0VmxSwitchVmcs(&pVCpu->hm.s.vmx.VmcsInfo, &pVCpu->hm.s.vmx.VmcsInfoNstGst);
10302	if (RT_LIKELY(rc == VINF_SUCCESS))
10303	{
10304	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = true;
10305	ASMSetFlags(fEFlags);
10306	pVmxTransient->pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10307
10308	/*
10309	* We use a different VM-exit MSR-store area for the nested-guest. Hence,
10310	* flag that we need to update the host MSR values there. Even if we decide
10311	* in the future to share the VM-exit MSR-store area page with the guest,
10312	* if its content differs, we would have to update the host MSRs anyway.
10313	*/
10314	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
10315	Assert(!pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer); /** @todo NSTVMX: Paranoia remove later. */
10316	}
10317	else
10318	{
10319	ASMSetFlags(fEFlags);
10320	return rc;
10321	}
10322	}
10323
10324	/*
10325	* Merge guest VMCS controls with the nested-guest VMCS controls.
10326	*
10327	* Even if we have not executed the guest prior to this (e.g. when resuming
10328	* from a saved state), we should be okay with merging controls as we
10329	* initialize the guest VMCS controls as part of VM setup phase.
10330	*/
10331	if (!pVCpu->hm.s.vmx.fMergedNstGstCtls)
10332	{
10333	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10334	AssertRCReturn(rc, rc);
10335	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10336	}
10337	}
10338	#endif
10339
10340	/*
10341	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10342	* We look at the guest VMCS control here as we always set it when supported by
10343	* the physical CPU. Looking at the nested-guest control here would not be
10344	* possible because they are not merged yet.
10345	*/
10346	PVM pVM = pVCpu->CTX_SUFF(pVM);
10347	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10348	Assert(pVmcsInfo);
10349	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10350	&& (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10351	&& PDMHasApic(pVM))
10352	{
10353	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10354	AssertRCReturn(rc, rc);
10355	}
10356
10357	/*
10358	* Evaluate events to be injected into the guest.
10359	*
10360	* Events in TRPM can be injected without inspecting the guest state.
10361	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10362	* guest to cause a VM-exit the next time they are ready to receive the event.
10363	*/
10364	if (TRPMHasTrap(pVCpu))
10365	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10366
10367	uint32_t fIntrState;
10368	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10369
10370	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10371	/*
10372	* While evaluating pending events if something failed (unlikely) or if we were
10373	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10374	*/
10375	if ( rcStrict != VINF_SUCCESS
10376	\|\| ( pVmxTransient->fIsNestedGuest
10377	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)))
10378	return rcStrict;
10379	#endif
10380
10381	/*
10382	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10383	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10384	* also result in triple-faulting the VM.
10385	*
10386	* The above does not apply when executing a nested-guest (since unrestricted guest execution
10387	* is a requirement) regardless doing it avoid duplicating code elsewhere.
10388	*/
10389	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10390	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10391	{ /* likely */ }
10392	else
10393	{
10394	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10395	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10396	return rcStrict;
10397	}
10398
10399	/*
10400	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10401	* import CR3 themselves. We will need to update them here, as even as late as the above
10402	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10403	* the below force flags to be set.
10404	*/
10405	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10406	{
10407	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10408	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10409	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10410	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10411	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10412	}
10413	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10414	{
10415	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10416	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10417	}
10418
10419	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10420	/* Paranoia. */
10421	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10422	#endif
10423
10424	/*
10425	* No longjmps to ring-3 from this point on!!!
10426	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10427	* This also disables flushing of the R0-logger instance (if any).
10428	*/
10429	VMMRZCallRing3Disable(pVCpu);
10430
10431	/*
10432	* Export the guest state bits.
10433	*
10434	* We cannot perform longjmps while loading the guest state because we do not preserve the
10435	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10436	* CPU migration.
10437	*
10438	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10439	* registers. Hence, loading of the guest state needs to be done -after- injection of events.
10440	*/
10441	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10442	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10443	{ /* likely */ }
10444	else
10445	{
10446	VMMRZCallRing3Enable(pVCpu);
10447	return rcStrict;
10448	}
10449
10450	/*
10451	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10452	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10453	* preemption disabled for a while. Since this is purely to aid the
10454	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10455	* disable interrupt on NT.
10456	*
10457	* We need to check for force-flags that could've possible been altered since we last
10458	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10459	* see @bugref{6398}).
10460	*
10461	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10462	* to ring-3 before executing guest code.
10463	*/
10464	pVmxTransient->fEFlags = ASMIntDisableFlags();
10465
10466	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10467	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10468	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10469	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10470	{
10471	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10472	{
10473	pVCpu->hm.s.Event.fPending = false;
10474
10475	/*
10476	* We've injected any pending events. This is really the point of no return (to ring-3).
10477	*
10478	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10479	* returns from this function, so don't enable them here.
10480	*/
10481	return VINF_SUCCESS;
10482	}
10483
10484	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10485	rcStrict = VINF_EM_RAW_INTERRUPT;
10486	}
10487	else
10488	{
10489	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10490	rcStrict = VINF_EM_RAW_TO_R3;
10491	}
10492
10493	ASMSetFlags(pVmxTransient->fEFlags);
10494	VMMRZCallRing3Enable(pVCpu);
10495
10496	return rcStrict;
10497	}
10498
10499
10500	/**
10501	* Final preparations before executing guest code using hardware-assisted VMX.
10502	*
10503	* We can no longer get preempted to a different host CPU and there are no returns
10504	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10505	* failures), this function is not intended to fail sans unrecoverable hardware
10506	* errors.
10507	*
10508	* @param pVCpu The cross context virtual CPU structure.
10509	* @param pVmxTransient The VMX-transient structure.
10510	*
10511	* @remarks Called with preemption disabled.
10512	* @remarks No-long-jump zone!!!
10513	*/
10514	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10515	{
10516	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10517	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10518	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10519	Assert(!pVCpu->hm.s.Event.fPending);
10520
10521	/*
10522	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10523	*/
10524	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10525	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10526
10527	PVM pVM = pVCpu->CTX_SUFF(pVM);
10528	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10529
10530	if (!CPUMIsGuestFPUStateActive(pVCpu))
10531	{
10532	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10533	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10534	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10535	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10536	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10537	}
10538
10539	/*
10540	* Re-save the host state bits as we may've been preempted (only happens when
10541	* thread-context hooks are used or when the VM start function changes).
10542	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10543	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10544	* see @bugref{8432}.
10545	*
10546	* This may also happen when switching to/from a nested-guest VMCS without leaving
10547	* ring-0.
10548	*/
10549	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10550	{
10551	int rc = hmR0VmxExportHostState(pVCpu);
10552	AssertRC(rc);
10553	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
10554	}
10555	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10556
10557	/*
10558	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10559	*/
10560	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10561	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10562	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10563
10564	/*
10565	* Store status of the shared guest/host debug state at the time of VM-entry.
10566	*/
10567	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
10568	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
10569	{
10570	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
10571	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
10572	}
10573	else
10574	#endif
10575	{
10576	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10577	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10578	}
10579
10580	/*
10581	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10582	* more than one conditional check. The post-run side of our code shall determine
10583	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10584	*/
10585	if (pVmcsInfo->pbVirtApic)
10586	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10587
10588	/*
10589	* Update the host MSRs values in the VM-exit MSR-load area.
10590	*/
10591	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10592	{
10593	if (pVmcsInfo->cExitMsrLoad > 0)
10594	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10595	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10596	}
10597
10598	/*
10599	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10600	* VMX-preemption timer based on the next virtual sync clock deadline.
10601	*/
10602	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10603	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10604	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10605	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10606	{
10607	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10608	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10609	}
10610
10611	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10612	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10613	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10614	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
10615
10616	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10617
10618	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10619	as we're about to start executing the guest . */
10620
10621	/*
10622	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10623	*
10624	* This is done this late as updating the TSC offsetting/preemption timer above
10625	* figures out if we can skip intercepting RDTSCP by calculating the number of
10626	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10627	*/
10628	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10629	{
10630	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10631	{
10632	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10633	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
10634	it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
10635	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10636	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10637	AssertRC(rc);
10638	}
10639	else
10640	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
10641	}
10642
10643	#ifdef VBOX_STRICT
10644	Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
10645	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo);
10646	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10647	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo));
10648	#endif
10649
10650	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10651	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10652	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs. */
10653	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10654	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10655	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10656	#endif
10657	}
10658
10659
10660	/**
10661	* First C routine invoked after running guest code using hardware-assisted VMX.
10662	*
10663	* @param pVCpu The cross context virtual CPU structure.
10664	* @param pVmxTransient The VMX-transient structure.
10665	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10666	*
10667	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10668	*
10669	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10670	* unconditionally when it is safe to do so.
10671	*/
10672	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10673	{
10674	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10675
10676	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10677	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10678	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10679	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10680	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10681	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10682
10683	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10684	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10685	{
10686	uint64_t uGstTsc;
10687	if (!pVmxTransient->fIsNestedGuest)
10688	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10689	else
10690	{
10691	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10692	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10693	}
10694	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10695	}
10696
10697	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10698	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
10699	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10700
10701	#if HC_ARCH_BITS == 64
10702	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10703	#endif
10704	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
10705	/* The 64-on-32 switcher maintains VMCS-launch state on its own
10706	and we need to leave it alone here. */
10707	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
10708	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10709	#else
10710	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10711	#endif
10712	#ifdef VBOX_STRICT
10713	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10714	#endif
10715	Assert(!ASMIntAreEnabled());
10716	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10717	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10718
10719	/*
10720	* Save the basic VM-exit reason and check if the VM-entry failed.
10721	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10722	*/
10723	uint32_t uExitReason;
10724	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
10725	AssertRC(rc);
10726	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
10727	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
10728
10729	/*
10730	* Check if VMLAUNCH/VMRESUME succeeded.
10731	* If this failed, we cause a guru meditation and cease further execution.
10732	*
10733	* However, if we are executing a nested-guest we might fail if we use the
10734	* fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
10735	*/
10736	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
10737	{
10738	/*
10739	* Update the VM-exit history array here even if the VM-entry failed due to:
10740	* - Invalid guest state.
10741	* - MSR loading.
10742	* - Machine-check event.
10743	*
10744	* In any of the above cases we will still have a "valid" VM-exit reason
10745	* despite @a fVMEntryFailed being false.
10746	*
10747	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
10748	*
10749	* Note! We don't have CS or RIP at this point. Will probably address that later
10750	* by amending the history entry added here.
10751	*/
10752	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
10753	UINT64_MAX, uHostTsc);
10754
10755	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
10756	{
10757	VMMRZCallRing3Enable(pVCpu);
10758
10759	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10760	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10761
10762	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
10763	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10764	AssertRC(rc);
10765	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
10766	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
10767	AssertRC(rc);
10768	#else
10769	/*
10770	* Import the guest-interruptibility state always as we need it while evaluating
10771	* injecting events on re-entry.
10772	*
10773	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
10774	* checking for real-mode while exporting the state because all bits that cause
10775	* mode changes wrt CR0 are intercepted.
10776	*/
10777	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
10778	AssertRC(rc);
10779	#endif
10780
10781	/*
10782	* Sync the TPR shadow with our APIC state.
10783	*/
10784	if ( !pVmxTransient->fIsNestedGuest
10785	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
10786	{
10787	Assert(pVmcsInfo->pbVirtApic);
10788	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
10789	{
10790	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
10791	AssertRC(rc);
10792	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
10793	}
10794	}
10795
10796	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10797	return;
10798	}
10799	}
10800	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10801	else if (pVmxTransient->fIsNestedGuest)
10802	{
10803	# if 0
10804	/*
10805	* Copy the VM-instruction error field to the guest VMCS.
10806	*/
10807	/** @todo NSTVMX: Verify we're using the fast path. */
10808	uint32_t u32RoVmInstrError;
10809	rc = VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &u32RoVmInstrError);
10810	AssertRCReturn(rc, rc);
10811	PVMXVVMCS pGstVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10812	pGstVmcs->u32RoVmInstrError = u32RoVmInstrError;
10813	/** @todo NSTVMX: Advance guest RIP and other fast path related restoration. */
10814	# else
10815	AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
10816	# endif
10817	}
10818	#endif
10819	else
10820	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
10821
10822	VMMRZCallRing3Enable(pVCpu);
10823	}
10824
10825
10826	/**
10827	* Runs the guest code using hardware-assisted VMX the normal way.
10828	*
10829	* @returns VBox status code.
10830	* @param pVCpu The cross context virtual CPU structure.
10831	* @param pcLoops Pointer to the number of executed loops.
10832	*/
10833	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
10834	{
10835	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10836	Assert(pcLoops);
10837	Assert(*pcLoops <= cMaxResumeLoops);
10838
10839	VMXTRANSIENT VmxTransient;
10840	RT_ZERO(VmxTransient);
10841	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10842
10843	/* Paranoia. */
10844	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
10845	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10846
10847	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10848	for (;;)
10849	{
10850	Assert(!HMR0SuspendPending());
10851	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10852	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10853
10854	/*
10855	* Preparatory work for running nested-guest code, this may force us to
10856	* return to ring-3.
10857	*
10858	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10859	*/
10860	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10861	if (rcStrict != VINF_SUCCESS)
10862	break;
10863
10864	/* Interrupts are disabled at this point! */
10865	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10866	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10867	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10868	/* Interrupts are re-enabled at this point! */
10869
10870	/*
10871	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10872	*/
10873	if (RT_SUCCESS(rcRun))
10874	{ /* very likely */ }
10875	else
10876	{
10877	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10878	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10879	return rcRun;
10880	}
10881
10882	/*
10883	* Profile the VM-exit.
10884	*/
10885	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10886	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10887	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10888	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10889	HMVMX_START_EXIT_DISPATCH_PROF();
10890
10891	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10892
10893	/*
10894	* Handle the VM-exit.
10895	*/
10896	#ifdef HMVMX_USE_FUNCTION_TABLE
10897	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
10898	#else
10899	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
10900	#endif
10901	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10902	if (rcStrict == VINF_SUCCESS)
10903	{
10904	if (++(*pcLoops) <= cMaxResumeLoops)
10905	continue;
10906	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10907	rcStrict = VINF_EM_RAW_INTERRUPT;
10908	}
10909	break;
10910	}
10911
10912	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10913	return rcStrict;
10914	}
10915
10916	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10917	/**
10918	* Runs the nested-guest code using hardware-assisted VMX.
10919	*
10920	* @returns VBox status code.
10921	* @param pVCpu The cross context virtual CPU structure.
10922	* @param pcLoops Pointer to the number of executed loops.
10923	*
10924	* @sa hmR0VmxRunGuestCodeNormal.
10925	*/
10926	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
10927	{
10928	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10929	Assert(pcLoops);
10930	Assert(*pcLoops <= cMaxResumeLoops);
10931
10932	VMXTRANSIENT VmxTransient;
10933	RT_ZERO(VmxTransient);
10934	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10935	VmxTransient.fIsNestedGuest = true;
10936
10937	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10938	for (;;)
10939	{
10940	Assert(!HMR0SuspendPending());
10941	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10942	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10943
10944	/*
10945	* Preparatory work for running guest code, this may force us to
10946	* return to ring-3.
10947	*
10948	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10949	*/
10950	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10951	if (rcStrict != VINF_SUCCESS)
10952	break;
10953
10954	/* Interrupts are disabled at this point! */
10955	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10956	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10957	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10958	/* Interrupts are re-enabled at this point! */
10959
10960	/*
10961	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10962	*/
10963	if (RT_SUCCESS(rcRun))
10964	{ /* very likely */ }
10965	else
10966	{
10967	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10968	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10969	return rcRun;
10970	}
10971
10972	/*
10973	* Profile the VM-exit.
10974	*/
10975	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10976	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10977	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10978	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10979	HMVMX_START_EXIT_DISPATCH_PROF();
10980
10981	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10982
10983	/*
10984	* Handle the VM-exit.
10985	*/
10986	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
10987	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10988	if ( rcStrict == VINF_SUCCESS
10989	&& CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10990	{
10991	if (++(*pcLoops) <= cMaxResumeLoops)
10992	continue;
10993	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10994	rcStrict = VINF_EM_RAW_INTERRUPT;
10995	}
10996	break;
10997	}
10998
10999	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11000	return rcStrict;
11001	}
11002	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11003
11004
11005	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
11006	* probes.
11007	*
11008	* The following few functions and associated structure contains the bloat
11009	* necessary for providing detailed debug events and dtrace probes as well as
11010	* reliable host side single stepping. This works on the principle of
11011	* "subclassing" the normal execution loop and workers. We replace the loop
11012	* method completely and override selected helpers to add necessary adjustments
11013	* to their core operation.
11014	*
11015	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11016	* any performance for debug and analysis features.
11017	*
11018	* @{
11019	*/
11020
11021	/**
11022	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11023	* the debug run loop.
11024	*/
11025	typedef struct VMXRUNDBGSTATE
11026	{
11027	/** The RIP we started executing at. This is for detecting that we stepped. */
11028	uint64_t uRipStart;
11029	/** The CS we started executing with. */
11030	uint16_t uCsStart;
11031
11032	/** Whether we've actually modified the 1st execution control field. */
11033	bool fModifiedProcCtls : 1;
11034	/** Whether we've actually modified the 2nd execution control field. */
11035	bool fModifiedProcCtls2 : 1;
11036	/** Whether we've actually modified the exception bitmap. */
11037	bool fModifiedXcptBitmap : 1;
11038
11039	/** We desire the modified the CR0 mask to be cleared. */
11040	bool fClearCr0Mask : 1;
11041	/** We desire the modified the CR4 mask to be cleared. */
11042	bool fClearCr4Mask : 1;
11043	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11044	uint32_t fCpe1Extra;
11045	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11046	uint32_t fCpe1Unwanted;
11047	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11048	uint32_t fCpe2Extra;
11049	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11050	uint32_t bmXcptExtra;
11051	/** The sequence number of the Dtrace provider settings the state was
11052	* configured against. */
11053	uint32_t uDtraceSettingsSeqNo;
11054	/** VM-exits to check (one bit per VM-exit). */
11055	uint32_t bmExitsToCheck[3];
11056
11057	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11058	uint32_t fProcCtlsInitial;
11059	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11060	uint32_t fProcCtls2Initial;
11061	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11062	uint32_t bmXcptInitial;
11063	} VMXRUNDBGSTATE;
11064	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11065	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11066
11067
11068	/**
11069	* Initializes the VMXRUNDBGSTATE structure.
11070	*
11071	* @param pVCpu The cross context virtual CPU structure of the
11072	* calling EMT.
11073	* @param pVmxTransient The VMX-transient structure.
11074	* @param pDbgState The debug state to initialize.
11075	*/
11076	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11077	{
11078	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11079	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11080
11081	pDbgState->fModifiedProcCtls = false;
11082	pDbgState->fModifiedProcCtls2 = false;
11083	pDbgState->fModifiedXcptBitmap = false;
11084	pDbgState->fClearCr0Mask = false;
11085	pDbgState->fClearCr4Mask = false;
11086	pDbgState->fCpe1Extra = 0;
11087	pDbgState->fCpe1Unwanted = 0;
11088	pDbgState->fCpe2Extra = 0;
11089	pDbgState->bmXcptExtra = 0;
11090	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11091	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11092	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11093	}
11094
11095
11096	/**
11097	* Updates the VMSC fields with changes requested by @a pDbgState.
11098	*
11099	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11100	* immediately before executing guest code, i.e. when interrupts are disabled.
11101	* We don't check status codes here as we cannot easily assert or return in the
11102	* latter case.
11103	*
11104	* @param pVCpu The cross context virtual CPU structure.
11105	* @param pVmxTransient The VMX-transient structure.
11106	* @param pDbgState The debug state.
11107	*/
11108	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11109	{
11110	/*
11111	* Ensure desired flags in VMCS control fields are set.
11112	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11113	*
11114	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11115	* there should be no stale data in pCtx at this point.
11116	*/
11117	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11118	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11119	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11120	{
11121	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11122	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11123	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11124	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11125	pDbgState->fModifiedProcCtls = true;
11126	}
11127
11128	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11129	{
11130	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11131	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11132	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11133	pDbgState->fModifiedProcCtls2 = true;
11134	}
11135
11136	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11137	{
11138	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11139	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11140	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11141	pDbgState->fModifiedXcptBitmap = true;
11142	}
11143
11144	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11145	{
11146	pVmcsInfo->u64Cr0Mask = 0;
11147	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
11148	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11149	}
11150
11151	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11152	{
11153	pVmcsInfo->u64Cr4Mask = 0;
11154	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
11155	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11156	}
11157
11158	NOREF(pVCpu);
11159	}
11160
11161
11162	/**
11163	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11164	* re-entry next time around.
11165	*
11166	* @returns Strict VBox status code (i.e. informational status codes too).
11167	* @param pVCpu The cross context virtual CPU structure.
11168	* @param pVmxTransient The VMX-transient structure.
11169	* @param pDbgState The debug state.
11170	* @param rcStrict The return code from executing the guest using single
11171	* stepping.
11172	*/
11173	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11174	VBOXSTRICTRC rcStrict)
11175	{
11176	/*
11177	* Restore VM-exit control settings as we may not reenter this function the
11178	* next time around.
11179	*/
11180	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11181
11182	/* We reload the initial value, trigger what we can of recalculations the
11183	next time around. From the looks of things, that's all that's required atm. */
11184	if (pDbgState->fModifiedProcCtls)
11185	{
11186	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11187	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11188	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11189	AssertRCReturn(rc2, rc2);
11190	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11191	}
11192
11193	/* We're currently the only ones messing with this one, so just restore the
11194	cached value and reload the field. */
11195	if ( pDbgState->fModifiedProcCtls2
11196	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11197	{
11198	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11199	AssertRCReturn(rc2, rc2);
11200	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11201	}
11202
11203	/* If we've modified the exception bitmap, we restore it and trigger
11204	reloading and partial recalculation the next time around. */
11205	if (pDbgState->fModifiedXcptBitmap)
11206	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11207
11208	return rcStrict;
11209	}
11210
11211
11212	/**
11213	* Configures VM-exit controls for current DBGF and DTrace settings.
11214	*
11215	* This updates @a pDbgState and the VMCS execution control fields to reflect
11216	* the necessary VM-exits demanded by DBGF and DTrace.
11217	*
11218	* @param pVCpu The cross context virtual CPU structure.
11219	* @param pVmxTransient The VMX-transient structure. May update
11220	* fUpdatedTscOffsettingAndPreemptTimer.
11221	* @param pDbgState The debug state.
11222	*/
11223	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11224	{
11225	/*
11226	* Take down the dtrace serial number so we can spot changes.
11227	*/
11228	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11229	ASMCompilerBarrier();
11230
11231	/*
11232	* We'll rebuild most of the middle block of data members (holding the
11233	* current settings) as we go along here, so start by clearing it all.
11234	*/
11235	pDbgState->bmXcptExtra = 0;
11236	pDbgState->fCpe1Extra = 0;
11237	pDbgState->fCpe1Unwanted = 0;
11238	pDbgState->fCpe2Extra = 0;
11239	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11240	pDbgState->bmExitsToCheck[i] = 0;
11241
11242	/*
11243	* Software interrupts (INT XXh) - no idea how to trigger these...
11244	*/
11245	PVM pVM = pVCpu->CTX_SUFF(pVM);
11246	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11247	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11248	{
11249	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11250	}
11251
11252	/*
11253	* INT3 breakpoints - triggered by #BP exceptions.
11254	*/
11255	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11256	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11257
11258	/*
11259	* Exception bitmap and XCPT events+probes.
11260	*/
11261	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11262	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11263	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11264
11265	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11266	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11267	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11268	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11269	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11270	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11271	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11272	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11273	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11274	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11275	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11276	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11277	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11278	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11279	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11280	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11281	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11282	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11283
11284	if (pDbgState->bmXcptExtra)
11285	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11286
11287	/*
11288	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11289	*
11290	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11291	* So, when adding/changing/removing please don't forget to update it.
11292	*
11293	* Some of the macros are picking up local variables to save horizontal space,
11294	* (being able to see it in a table is the lesser evil here).
11295	*/
11296	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11297	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11298	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11299	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11300	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11301	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11302	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11303	} else do { } while (0)
11304	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11305	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11306	{ \
11307	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11308	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11309	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11310	} else do { } while (0)
11311	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11312	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11313	{ \
11314	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11315	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11316	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11317	} else do { } while (0)
11318	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11319	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11320	{ \
11321	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11322	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11323	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11324	} else do { } while (0)
11325
11326	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11327	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11328	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11329	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11330	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11331
11332	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11333	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11334	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11335	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11336	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11337	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11338	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11339	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11340	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11341	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11342	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11343	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11344	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11345	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11346	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11347	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11348	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11349	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11350	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11351	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11352	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11353	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11354	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11355	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11356	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11357	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11358	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11359	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11360	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11361	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11362	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11363	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11364	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11365	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11366	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11367	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11368
11369	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11370	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11371	{
11372	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11373	\| CPUMCTX_EXTRN_APIC_TPR);
11374	AssertRC(rc);
11375
11376	#if 0 /** @todo fix me */
11377	pDbgState->fClearCr0Mask = true;
11378	pDbgState->fClearCr4Mask = true;
11379	#endif
11380	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11381	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11382	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11383	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11384	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11385	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11386	require clearing here and in the loop if we start using it. */
11387	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11388	}
11389	else
11390	{
11391	if (pDbgState->fClearCr0Mask)
11392	{
11393	pDbgState->fClearCr0Mask = false;
11394	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11395	}
11396	if (pDbgState->fClearCr4Mask)
11397	{
11398	pDbgState->fClearCr4Mask = false;
11399	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11400	}
11401	}
11402	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11403	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11404
11405	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11406	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11407	{
11408	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11409	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11410	}
11411	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11412	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11413
11414	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11415	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11416	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11417	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11418	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11419	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11420	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11421	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11422	#if 0 /** @todo too slow, fix handler. */
11423	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11424	#endif
11425	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11426
11427	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11428	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11429	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11430	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11431	{
11432	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11433	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11434	}
11435	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11436	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11437	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11438	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11439
11440	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11441	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11442	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11443	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11444	{
11445	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11446	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11447	}
11448	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11449	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11450	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11451	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11452
11453	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11454	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11455	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11456	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11457	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11458	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11459	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11460	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11461	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11462	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11463	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11464	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11465	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11466	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11467	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11468	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11469	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11470	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11471	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11472	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11473	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11474	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11475
11476	#undef IS_EITHER_ENABLED
11477	#undef SET_ONLY_XBM_IF_EITHER_EN
11478	#undef SET_CPE1_XBM_IF_EITHER_EN
11479	#undef SET_CPEU_XBM_IF_EITHER_EN
11480	#undef SET_CPE2_XBM_IF_EITHER_EN
11481
11482	/*
11483	* Sanitize the control stuff.
11484	*/
11485	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11486	if (pDbgState->fCpe2Extra)
11487	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11488	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11489	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11490	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11491	{
11492	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11493	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11494	}
11495
11496	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11497	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11498	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11499	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11500	}
11501
11502
11503	/**
11504	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11505	* appropriate.
11506	*
11507	* The caller has checked the VM-exit against the
11508	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11509	* already, so we don't have to do that either.
11510	*
11511	* @returns Strict VBox status code (i.e. informational status codes too).
11512	* @param pVCpu The cross context virtual CPU structure.
11513	* @param pVmxTransient The VMX-transient structure.
11514	* @param uExitReason The VM-exit reason.
11515	*
11516	* @remarks The name of this function is displayed by dtrace, so keep it short
11517	* and to the point. No longer than 33 chars long, please.
11518	*/
11519	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11520	{
11521	/*
11522	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11523	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11524	*
11525	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11526	* does. Must add/change/remove both places. Same ordering, please.
11527	*
11528	* Added/removed events must also be reflected in the next section
11529	* where we dispatch dtrace events.
11530	*/
11531	bool fDtrace1 = false;
11532	bool fDtrace2 = false;
11533	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11534	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11535	uint32_t uEventArg = 0;
11536	#define SET_EXIT(a_EventSubName) \
11537	do { \
11538	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11539	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11540	} while (0)
11541	#define SET_BOTH(a_EventSubName) \
11542	do { \
11543	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11544	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11545	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11546	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11547	} while (0)
11548	switch (uExitReason)
11549	{
11550	case VMX_EXIT_MTF:
11551	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11552
11553	case VMX_EXIT_XCPT_OR_NMI:
11554	{
11555	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11556	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11557	{
11558	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11559	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11560	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11561	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11562	{
11563	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11564	{
11565	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11566	uEventArg = pVmxTransient->uExitIntErrorCode;
11567	}
11568	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11569	switch (enmEvent1)
11570	{
11571	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11572	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11573	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11574	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11575	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11576	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11577	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11578	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11579	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11580	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11581	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11582	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11583	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11584	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11585	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11586	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11587	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11588	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11589	default: break;
11590	}
11591	}
11592	else
11593	AssertFailed();
11594	break;
11595
11596	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11597	uEventArg = idxVector;
11598	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11599	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11600	break;
11601	}
11602	break;
11603	}
11604
11605	case VMX_EXIT_TRIPLE_FAULT:
11606	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11607	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11608	break;
11609	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11610	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11611	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11612	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11613	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11614
11615	/* Instruction specific VM-exits: */
11616	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11617	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11618	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11619	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11620	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11621	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11622	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11623	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11624	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11625	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11626	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11627	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11628	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11629	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11630	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11631	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11632	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11633	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11634	case VMX_EXIT_MOV_CRX:
11635	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11636	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11637	SET_BOTH(CRX_READ);
11638	else
11639	SET_BOTH(CRX_WRITE);
11640	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11641	break;
11642	case VMX_EXIT_MOV_DRX:
11643	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11644	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11645	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11646	SET_BOTH(DRX_READ);
11647	else
11648	SET_BOTH(DRX_WRITE);
11649	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11650	break;
11651	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11652	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11653	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11654	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11655	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11656	case VMX_EXIT_GDTR_IDTR_ACCESS:
11657	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11658	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11659	{
11660	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11661	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11662	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11663	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11664	}
11665	break;
11666
11667	case VMX_EXIT_LDTR_TR_ACCESS:
11668	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11669	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11670	{
11671	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11672	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11673	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11674	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11675	}
11676	break;
11677
11678	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11679	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11680	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11681	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11682	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11683	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11684	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11685	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11686	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11687	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11688	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11689
11690	/* Events that aren't relevant at this point. */
11691	case VMX_EXIT_EXT_INT:
11692	case VMX_EXIT_INT_WINDOW:
11693	case VMX_EXIT_NMI_WINDOW:
11694	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11695	case VMX_EXIT_PREEMPT_TIMER:
11696	case VMX_EXIT_IO_INSTR:
11697	break;
11698
11699	/* Errors and unexpected events. */
11700	case VMX_EXIT_INIT_SIGNAL:
11701	case VMX_EXIT_SIPI:
11702	case VMX_EXIT_IO_SMI:
11703	case VMX_EXIT_SMI:
11704	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11705	case VMX_EXIT_ERR_MSR_LOAD:
11706	case VMX_EXIT_ERR_MACHINE_CHECK:
11707	break;
11708
11709	default:
11710	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11711	break;
11712	}
11713	#undef SET_BOTH
11714	#undef SET_EXIT
11715
11716	/*
11717	* Dtrace tracepoints go first. We do them here at once so we don't
11718	* have to copy the guest state saving and stuff a few dozen times.
11719	* Down side is that we've got to repeat the switch, though this time
11720	* we use enmEvent since the probes are a subset of what DBGF does.
11721	*/
11722	if (fDtrace1 \|\| fDtrace2)
11723	{
11724	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11725	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11726	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11727	switch (enmEvent1)
11728	{
11729	/** @todo consider which extra parameters would be helpful for each probe. */
11730	case DBGFEVENT_END: break;
11731	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11732	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11733	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11734	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11735	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11736	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11737	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11738	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11739	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
11740	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
11741	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
11742	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
11743	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
11744	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
11745	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
11746	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
11747	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
11748	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
11749	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11750	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11751	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
11752	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
11753	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
11754	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
11755	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
11756	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
11757	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
11758	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11759	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11760	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11761	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11762	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11763	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
11764	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11765	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
11766	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
11767	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
11768	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
11769	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
11770	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
11771	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
11772	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
11773	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
11774	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
11775	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
11776	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
11777	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
11778	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
11779	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
11780	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
11781	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
11782	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
11783	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
11784	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
11785	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
11786	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
11787	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
11788	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
11789	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
11790	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
11791	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
11792	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
11793	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
11794	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
11795	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
11796	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
11797	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
11798	}
11799	switch (enmEvent2)
11800	{
11801	/** @todo consider which extra parameters would be helpful for each probe. */
11802	case DBGFEVENT_END: break;
11803	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
11804	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11805	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
11806	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
11807	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
11808	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
11809	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
11810	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
11811	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
11812	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11813	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11814	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11815	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11816	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11817	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
11818	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11819	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
11820	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
11821	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
11822	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
11823	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
11824	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
11825	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
11826	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
11827	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
11828	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
11829	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
11830	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
11831	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
11832	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
11833	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
11834	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
11835	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
11836	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
11837	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
11838	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
11839	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
11840	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
11841	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
11842	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
11843	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
11844	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
11845	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
11846	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
11847	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
11848	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
11849	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
11850	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
11851	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
11852	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
11853	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
11854	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
11855	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
11856	}
11857	}
11858
11859	/*
11860	* Fire of the DBGF event, if enabled (our check here is just a quick one,
11861	* the DBGF call will do a full check).
11862	*
11863	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
11864	* Note! If we have to events, we prioritize the first, i.e. the instruction
11865	* one, in order to avoid event nesting.
11866	*/
11867	PVM pVM = pVCpu->CTX_SUFF(pVM);
11868	if ( enmEvent1 != DBGFEVENT_END
11869	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
11870	{
11871	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11872	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
11873	if (rcStrict != VINF_SUCCESS)
11874	return rcStrict;
11875	}
11876	else if ( enmEvent2 != DBGFEVENT_END
11877	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
11878	{
11879	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11880	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
11881	if (rcStrict != VINF_SUCCESS)
11882	return rcStrict;
11883	}
11884
11885	return VINF_SUCCESS;
11886	}
11887
11888
11889	/**
11890	* Single-stepping VM-exit filtering.
11891	*
11892	* This is preprocessing the VM-exits and deciding whether we've gotten far
11893	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
11894	* handling is performed.
11895	*
11896	* @returns Strict VBox status code (i.e. informational status codes too).
11897	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
11898	* @param pVmxTransient The VMX-transient structure.
11899	* @param pDbgState The debug state.
11900	*/
11901	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11902	{
11903	/*
11904	* Expensive (saves context) generic dtrace VM-exit probe.
11905	*/
11906	uint32_t const uExitReason = pVmxTransient->uExitReason;
11907	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
11908	{ /* more likely */ }
11909	else
11910	{
11911	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11912	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11913	AssertRC(rc);
11914	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
11915	}
11916
11917	/*
11918	* Check for host NMI, just to get that out of the way.
11919	*/
11920	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
11921	{ /* normally likely */ }
11922	else
11923	{
11924	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11925	AssertRCReturn(rc2, rc2);
11926	uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11927	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11928	return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
11929	}
11930
11931	/*
11932	* Check for single stepping event if we're stepping.
11933	*/
11934	if (pVCpu->hm.s.fSingleInstruction)
11935	{
11936	switch (uExitReason)
11937	{
11938	case VMX_EXIT_MTF:
11939	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11940
11941	/* Various events: */
11942	case VMX_EXIT_XCPT_OR_NMI:
11943	case VMX_EXIT_EXT_INT:
11944	case VMX_EXIT_TRIPLE_FAULT:
11945	case VMX_EXIT_INT_WINDOW:
11946	case VMX_EXIT_NMI_WINDOW:
11947	case VMX_EXIT_TASK_SWITCH:
11948	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11949	case VMX_EXIT_APIC_ACCESS:
11950	case VMX_EXIT_EPT_VIOLATION:
11951	case VMX_EXIT_EPT_MISCONFIG:
11952	case VMX_EXIT_PREEMPT_TIMER:
11953
11954	/* Instruction specific VM-exits: */
11955	case VMX_EXIT_CPUID:
11956	case VMX_EXIT_GETSEC:
11957	case VMX_EXIT_HLT:
11958	case VMX_EXIT_INVD:
11959	case VMX_EXIT_INVLPG:
11960	case VMX_EXIT_RDPMC:
11961	case VMX_EXIT_RDTSC:
11962	case VMX_EXIT_RSM:
11963	case VMX_EXIT_VMCALL:
11964	case VMX_EXIT_VMCLEAR:
11965	case VMX_EXIT_VMLAUNCH:
11966	case VMX_EXIT_VMPTRLD:
11967	case VMX_EXIT_VMPTRST:
11968	case VMX_EXIT_VMREAD:
11969	case VMX_EXIT_VMRESUME:
11970	case VMX_EXIT_VMWRITE:
11971	case VMX_EXIT_VMXOFF:
11972	case VMX_EXIT_VMXON:
11973	case VMX_EXIT_MOV_CRX:
11974	case VMX_EXIT_MOV_DRX:
11975	case VMX_EXIT_IO_INSTR:
11976	case VMX_EXIT_RDMSR:
11977	case VMX_EXIT_WRMSR:
11978	case VMX_EXIT_MWAIT:
11979	case VMX_EXIT_MONITOR:
11980	case VMX_EXIT_PAUSE:
11981	case VMX_EXIT_GDTR_IDTR_ACCESS:
11982	case VMX_EXIT_LDTR_TR_ACCESS:
11983	case VMX_EXIT_INVEPT:
11984	case VMX_EXIT_RDTSCP:
11985	case VMX_EXIT_INVVPID:
11986	case VMX_EXIT_WBINVD:
11987	case VMX_EXIT_XSETBV:
11988	case VMX_EXIT_RDRAND:
11989	case VMX_EXIT_INVPCID:
11990	case VMX_EXIT_VMFUNC:
11991	case VMX_EXIT_RDSEED:
11992	case VMX_EXIT_XSAVES:
11993	case VMX_EXIT_XRSTORS:
11994	{
11995	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11996	AssertRCReturn(rc, rc);
11997	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
11998	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
11999	return VINF_EM_DBG_STEPPED;
12000	break;
12001	}
12002
12003	/* Errors and unexpected events: */
12004	case VMX_EXIT_INIT_SIGNAL:
12005	case VMX_EXIT_SIPI:
12006	case VMX_EXIT_IO_SMI:
12007	case VMX_EXIT_SMI:
12008	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12009	case VMX_EXIT_ERR_MSR_LOAD:
12010	case VMX_EXIT_ERR_MACHINE_CHECK:
12011	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12012	break;
12013
12014	default:
12015	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12016	break;
12017	}
12018	}
12019
12020	/*
12021	* Check for debugger event breakpoints and dtrace probes.
12022	*/
12023	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12024	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12025	{
12026	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12027	if (rcStrict != VINF_SUCCESS)
12028	return rcStrict;
12029	}
12030
12031	/*
12032	* Normal processing.
12033	*/
12034	#ifdef HMVMX_USE_FUNCTION_TABLE
12035	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
12036	#else
12037	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
12038	#endif
12039	}
12040
12041
12042	/**
12043	* Single steps guest code using hardware-assisted VMX.
12044	*
12045	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
12046	* but single-stepping through the hypervisor debugger.
12047	*
12048	* @returns Strict VBox status code (i.e. informational status codes too).
12049	* @param pVCpu The cross context virtual CPU structure.
12050	* @param pcLoops Pointer to the number of executed loops.
12051	*
12052	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
12053	*/
12054	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
12055	{
12056	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
12057	Assert(pcLoops);
12058	Assert(*pcLoops <= cMaxResumeLoops);
12059
12060	VMXTRANSIENT VmxTransient;
12061	RT_ZERO(VmxTransient);
12062	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
12063
12064	/* Set HMCPU indicators. */
12065	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
12066	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
12067	pVCpu->hm.s.fDebugWantRdTscExit = false;
12068	pVCpu->hm.s.fUsingDebugLoop = true;
12069
12070	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
12071	VMXRUNDBGSTATE DbgState;
12072	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
12073	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12074
12075	/*
12076	* The loop.
12077	*/
12078	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
12079	for (;;)
12080	{
12081	Assert(!HMR0SuspendPending());
12082	HMVMX_ASSERT_CPU_SAFE(pVCpu);
12083	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12084	bool fStepping = pVCpu->hm.s.fSingleInstruction;
12085
12086	/* Set up VM-execution controls the next two can respond to. */
12087	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12088
12089	/*
12090	* Preparatory work for running guest code, this may force us to
12091	* return to ring-3.
12092	*
12093	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12094	*/
12095	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12096	if (rcStrict != VINF_SUCCESS)
12097	break;
12098
12099	/* Interrupts are disabled at this point! */
12100	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12101
12102	/* Override any obnoxious code in the above two calls. */
12103	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12104
12105	/*
12106	* Finally execute the guest.
12107	*/
12108	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12109
12110	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12111	/* Interrupts are re-enabled at this point! */
12112
12113	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12114	if (RT_SUCCESS(rcRun))
12115	{ /* very likely */ }
12116	else
12117	{
12118	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12119	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12120	return rcRun;
12121	}
12122
12123	/* Profile the VM-exit. */
12124	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12125	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12126	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12127	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12128	HMVMX_START_EXIT_DISPATCH_PROF();
12129
12130	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12131
12132	/*
12133	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12134	*/
12135	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12136	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12137	if (rcStrict != VINF_SUCCESS)
12138	break;
12139	if (++(*pcLoops) > cMaxResumeLoops)
12140	{
12141	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12142	rcStrict = VINF_EM_RAW_INTERRUPT;
12143	break;
12144	}
12145
12146	/*
12147	* Stepping: Did the RIP change, if so, consider it a single step.
12148	* Otherwise, make sure one of the TFs gets set.
12149	*/
12150	if (fStepping)
12151	{
12152	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12153	AssertRC(rc);
12154	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12155	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12156	{
12157	rcStrict = VINF_EM_DBG_STEPPED;
12158	break;
12159	}
12160	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12161	}
12162
12163	/*
12164	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12165	*/
12166	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12167	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12168	}
12169
12170	/*
12171	* Clear the X86_EFL_TF if necessary.
12172	*/
12173	if (pVCpu->hm.s.fClearTrapFlag)
12174	{
12175	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12176	AssertRC(rc);
12177	pVCpu->hm.s.fClearTrapFlag = false;
12178	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12179	}
12180	/** @todo there seems to be issues with the resume flag when the monitor trap
12181	* flag is pending without being used. Seen early in bios init when
12182	* accessing APIC page in protected mode. */
12183
12184	/*
12185	* Restore VM-exit control settings as we may not re-enter this function the
12186	* next time around.
12187	*/
12188	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12189
12190	/* Restore HMCPU indicators. */
12191	pVCpu->hm.s.fUsingDebugLoop = false;
12192	pVCpu->hm.s.fDebugWantRdTscExit = false;
12193	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12194
12195	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12196	return rcStrict;
12197	}
12198
12199
12200	/** @} */
12201
12202
12203	/**
12204	* Checks if any expensive dtrace probes are enabled and we should go to the
12205	* debug loop.
12206	*
12207	* @returns true if we should use debug loop, false if not.
12208	*/
12209	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12210	{
12211	/* It's probably faster to OR the raw 32-bit counter variables together.
12212	Since the variables are in an array and the probes are next to one
12213	another (more or less), we have good locality. So, better read
12214	eight-nine cache lines ever time and only have one conditional, than
12215	128+ conditionals, right? */
12216	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12217	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12218	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12219	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12220	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12221	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12222	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12223	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12224	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12225	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12226	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12227	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12228	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12229	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12230	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12231	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12232	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12233	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12234	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12235	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12236	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12237	) != 0
12238	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12239	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12240	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12241	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12242	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12243	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12244	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12245	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12246	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12247	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12248	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12249	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12250	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12251	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12252	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12253	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12254	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12255	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12256	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12257	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12258	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12259	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12260	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12261	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12262	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12263	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12264	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12265	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12266	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12267	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12268	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12269	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12270	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12271	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12272	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12273	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12274	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12275	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12276	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12277	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12278	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12279	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12280	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12281	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12282	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12283	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12284	) != 0
12285	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12286	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12287	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12288	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12289	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12290	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12291	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12292	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12293	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12294	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12295	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12296	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12297	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12298	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12299	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12300	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12301	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12302	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12303	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12304	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12305	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12306	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12307	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12308	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12309	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12310	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12311	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12312	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12313	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12314	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12315	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12316	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12317	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12318	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12319	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12320	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12321	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12322	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12323	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12324	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12325	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12326	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12327	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12328	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12329	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12330	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12331	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12332	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12333	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12334	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12335	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12336	) != 0;
12337	}
12338
12339
12340	/**
12341	* Runs the guest using hardware-assisted VMX.
12342	*
12343	* @returns Strict VBox status code (i.e. informational status codes too).
12344	* @param pVCpu The cross context virtual CPU structure.
12345	*/
12346	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
12347	{
12348	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12349	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12350	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12351	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12352
12353	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
12354
12355	VBOXSTRICTRC rcStrict;
12356	uint32_t cLoops = 0;
12357	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12358	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12359	#else
12360	bool const fInNestedGuestMode = false;
12361	#endif
12362	if (!fInNestedGuestMode)
12363	{
12364	if ( !pVCpu->hm.s.fUseDebugLoop
12365	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12366	&& !DBGFIsStepping(pVCpu)
12367	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12368	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12369	else
12370	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12371	}
12372	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12373	else
12374	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
12375
12376	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12377	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12378	#endif
12379
12380	if (rcStrict == VERR_EM_INTERPRETER)
12381	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12382	else if (rcStrict == VINF_EM_RESET)
12383	rcStrict = VINF_EM_TRIPLE_FAULT;
12384
12385	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12386	if (RT_FAILURE(rc2))
12387	{
12388	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12389	rcStrict = rc2;
12390	}
12391	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12392	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12393	return rcStrict;
12394	}
12395
12396
12397	#ifndef HMVMX_USE_FUNCTION_TABLE
12398	/**
12399	* Handles a guest VM-exit from hardware-assisted VMX execution.
12400	*
12401	* @returns Strict VBox status code (i.e. informational status codes too).
12402	* @param pVCpu The cross context virtual CPU structure.
12403	* @param pVmxTransient The VMX-transient structure.
12404	*/
12405	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12406	{
12407	#ifdef DEBUG_ramshankar
12408	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12409	do { \
12410	if (a_fSave != 0) \
12411	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
12412	VBOXSTRICTRC rcStrict = a_CallExpr; \
12413	if (a_fSave != 0) \
12414	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12415	return rcStrict; \
12416	} while (0)
12417	#else
12418	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12419	#endif
12420	uint32_t const rcReason = pVmxTransient->uExitReason;
12421	switch (rcReason)
12422	{
12423	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12424	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12425	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12426	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12427	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12428	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12429	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12430	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12431	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12432	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12433	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12434	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12435	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12436	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12437	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12438	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12439	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12440	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12441	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12442	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12443	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12444	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12445	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12446	case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
12447	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12448	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12449	case VMX_EXIT_GDTR_IDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12450	case VMX_EXIT_LDTR_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12451	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12452	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12453	case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
12454	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12455	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12456	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12457	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12458	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12459	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12460	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12461	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12462	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12463	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12464	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12465	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12466	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12467	#else
12468	case VMX_EXIT_VMCLEAR:
12469	case VMX_EXIT_VMLAUNCH:
12470	case VMX_EXIT_VMPTRLD:
12471	case VMX_EXIT_VMPTRST:
12472	case VMX_EXIT_VMREAD:
12473	case VMX_EXIT_VMRESUME:
12474	case VMX_EXIT_VMWRITE:
12475	case VMX_EXIT_VMXOFF:
12476	case VMX_EXIT_VMXON:
12477	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12478	#endif
12479
12480	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12481	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12482	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
12483	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
12484	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
12485	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
12486	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
12487	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12488	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
12489
12490	case VMX_EXIT_INVEPT:
12491	case VMX_EXIT_INVVPID:
12492	case VMX_EXIT_VMFUNC:
12493	case VMX_EXIT_XSAVES:
12494	case VMX_EXIT_XRSTORS:
12495	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12496
12497	case VMX_EXIT_ENCLS:
12498	case VMX_EXIT_RDSEED:
12499	case VMX_EXIT_PML_FULL:
12500	default:
12501	return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
12502	}
12503	#undef VMEXIT_CALL_RET
12504	}
12505	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12506
12507
12508	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12509	/**
12510	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12511	*
12512	* @returns Strict VBox status code (i.e. informational status codes too).
12513	* @param pVCpu The cross context virtual CPU structure.
12514	* @param pVmxTransient The VMX-transient structure.
12515	*/
12516	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12517	{
12518	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
12519	uint32_t const rcReason = pVmxTransient->uExitReason;
12520	switch (rcReason)
12521	{
12522	case VMX_EXIT_EPT_MISCONFIG:
12523	rcStrict = hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
12524	break;
12525
12526	case VMX_EXIT_EPT_VIOLATION:
12527	rcStrict = hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
12528	break;
12529
12530	case VMX_EXIT_IO_INSTR:
12531	{
12532	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12533	AssertRCReturn(rc, rc);
12534
12535	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
12536	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
12537	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
12538
12539	/* Size of the I/O accesses in bytes. */
12540	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 };
12541	uint8_t const cbAccess = s_aIOSizes[uIOSize];
12542
12543	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
12544	{
12545	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12546	AssertRCReturn(rc, rc);
12547
12548	VMXVEXITINFO ExitInfo;
12549	RT_ZERO(ExitInfo);
12550	ExitInfo.uReason = pVmxTransient->uExitReason;
12551	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12552	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12553	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12554	}
12555	else
12556	rcStrict = hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
12557	break;
12558	}
12559
12560	case VMX_EXIT_CPUID:
12561	case VMX_EXIT_RDTSC:
12562	case VMX_EXIT_RDTSCP:
12563	case VMX_EXIT_APIC_ACCESS:
12564	case VMX_EXIT_XCPT_OR_NMI:
12565	case VMX_EXIT_MOV_CRX:
12566	case VMX_EXIT_EXT_INT:
12567	case VMX_EXIT_INT_WINDOW:
12568	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12569	case VMX_EXIT_MWAIT:
12570	case VMX_EXIT_MONITOR:
12571	case VMX_EXIT_TASK_SWITCH:
12572	case VMX_EXIT_PREEMPT_TIMER:
12573	case VMX_EXIT_RDMSR:
12574	case VMX_EXIT_WRMSR:
12575	case VMX_EXIT_VMCALL:
12576	case VMX_EXIT_MOV_DRX:
12577	case VMX_EXIT_HLT:
12578	case VMX_EXIT_INVD:
12579	case VMX_EXIT_INVLPG:
12580	case VMX_EXIT_RSM:
12581	case VMX_EXIT_MTF:
12582	case VMX_EXIT_PAUSE:
12583	case VMX_EXIT_GDTR_IDTR_ACCESS:
12584	case VMX_EXIT_LDTR_TR_ACCESS:
12585	case VMX_EXIT_WBINVD:
12586	case VMX_EXIT_XSETBV:
12587	case VMX_EXIT_RDRAND:
12588	case VMX_EXIT_INVPCID:
12589	case VMX_EXIT_GETSEC:
12590	case VMX_EXIT_RDPMC:
12591	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12592	case VMX_EXIT_VMCLEAR:
12593	case VMX_EXIT_VMLAUNCH:
12594	case VMX_EXIT_VMPTRLD:
12595	case VMX_EXIT_VMPTRST:
12596	case VMX_EXIT_VMREAD:
12597	case VMX_EXIT_VMRESUME:
12598	case VMX_EXIT_VMWRITE:
12599	case VMX_EXIT_VMXOFF:
12600	case VMX_EXIT_VMXON:
12601	#endif
12602	case VMX_EXIT_TRIPLE_FAULT:
12603	case VMX_EXIT_NMI_WINDOW:
12604	case VMX_EXIT_INIT_SIGNAL:
12605	case VMX_EXIT_SIPI:
12606	case VMX_EXIT_IO_SMI:
12607	case VMX_EXIT_SMI:
12608	case VMX_EXIT_ERR_MSR_LOAD:
12609	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12610	case VMX_EXIT_ERR_MACHINE_CHECK:
12611
12612	case VMX_EXIT_INVEPT:
12613	case VMX_EXIT_INVVPID:
12614	case VMX_EXIT_VMFUNC:
12615	case VMX_EXIT_XSAVES:
12616	case VMX_EXIT_XRSTORS:
12617
12618	case VMX_EXIT_ENCLS:
12619	case VMX_EXIT_RDSEED:
12620	case VMX_EXIT_PML_FULL:
12621	default:
12622	return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
12623	}
12624
12625	return rcStrict;
12626	}
12627	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12628
12629
12630	#ifdef VBOX_STRICT
12631	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
12632	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
12633	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
12634
12635	# define HMVMX_ASSERT_PREEMPT_CPUID() \
12636	do { \
12637	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
12638	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
12639	} while (0)
12640
12641	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12642	do { \
12643	AssertPtr((a_pVCpu)); \
12644	AssertPtr((a_pVmxTransient)); \
12645	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
12646	Assert((a_pVmxTransient)->pVmcsInfo); \
12647	Assert(ASMIntAreEnabled()); \
12648	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12649	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
12650	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
12651	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12652	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
12653	HMVMX_ASSERT_PREEMPT_CPUID(); \
12654	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12655	} while (0)
12656
12657	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12658	do { \
12659	Log4Func(("\n")); \
12660	} while (0)
12661	#else
12662	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12663	do { \
12664	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12665	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
12666	} while (0)
12667	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
12668	#endif
12669
12670
12671	/**
12672	* Advances the guest RIP by the specified number of bytes.
12673	*
12674	* @param pVCpu The cross context virtual CPU structure.
12675	* @param cbInstr Number of bytes to advance the RIP by.
12676	*
12677	* @remarks No-long-jump zone!!!
12678	*/
12679	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
12680	{
12681	/* Advance the RIP. */
12682	pVCpu->cpum.GstCtx.rip += cbInstr;
12683	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12684
12685	/* Update interrupt inhibition. */
12686	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
12687	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
12688	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12689	}
12690
12691
12692	/**
12693	* Advances the guest RIP after reading it from the VMCS.
12694	*
12695	* @returns VBox status code, no informational status codes.
12696	* @param pVCpu The cross context virtual CPU structure.
12697	* @param pVmxTransient The VMX-transient structure.
12698	*
12699	* @remarks No-long-jump zone!!!
12700	*/
12701	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12702	{
12703	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12704	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
12705	AssertRCReturn(rc, rc);
12706
12707	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
12708	return VINF_SUCCESS;
12709	}
12710
12711
12712	/**
12713	* Handle a condition that occurred while delivering an event through the guest
12714	* IDT.
12715	*
12716	* @returns Strict VBox status code (i.e. informational status codes too).
12717	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
12718	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
12719	* to continue execution of the guest which will delivery the \#DF.
12720	* @retval VINF_EM_RESET if we detected a triple-fault condition.
12721	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
12722	*
12723	* @param pVCpu The cross context virtual CPU structure.
12724	* @param pVmxTransient The VMX-transient structure.
12725	*
12726	* @remarks No-long-jump zone!!!
12727	*/
12728	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12729	{
12730	uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12731
12732	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12733	rc2 \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12734	AssertRCReturn(rc2, rc2);
12735
12736	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
12737	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12738	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12739	{
12740	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12741	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12742
12743	/*
12744	* If the event was a software interrupt (generated with INT n) or a software exception
12745	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
12746	* can handle the VM-exit and continue guest execution which will re-execute the
12747	* instruction rather than re-injecting the exception, as that can cause premature
12748	* trips to ring-3 before injection and involve TRPM which currently has no way of
12749	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
12750	* the problem).
12751	*/
12752	IEMXCPTRAISE enmRaise;
12753	IEMXCPTRAISEINFO fRaiseInfo;
12754	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
12755	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
12756	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
12757	{
12758	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
12759	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12760	}
12761	else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
12762	{
12763	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12764	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
12765	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
12766	/** @todo Make AssertMsgReturn as just AssertMsg later. */
12767	AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
12768	("Unexpected VM-exit interruption vector type %#x!\n", uExitVectorType), VERR_VMX_IPE_5);
12769
12770	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
12771
12772	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
12773	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
12774	{
12775	pVmxTransient->fVectoringPF = true;
12776	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12777	}
12778	}
12779	else
12780	{
12781	/*
12782	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
12783	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
12784	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
12785	*/
12786	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12787	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12788	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
12789	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12790	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12791	}
12792
12793	/*
12794	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
12795	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
12796	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
12797	* subsequent VM-entry would fail.
12798	*
12799	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
12800	*/
12801	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS)
12802	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12803	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
12804	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
12805	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
12806	{
12807	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
12808	}
12809
12810	switch (enmRaise)
12811	{
12812	case IEMXCPTRAISE_CURRENT_XCPT:
12813	{
12814	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
12815	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
12816	Assert(rcStrict == VINF_SUCCESS);
12817	break;
12818	}
12819
12820	case IEMXCPTRAISE_PREV_EVENT:
12821	{
12822	uint32_t u32ErrCode;
12823	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
12824	{
12825	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12826	AssertRCReturn(rc2, rc2);
12827	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12828	}
12829	else
12830	u32ErrCode = 0;
12831
12832	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
12833	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
12834	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12835	0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
12836
12837	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
12838	pVCpu->hm.s.Event.u32ErrCode));
12839	Assert(rcStrict == VINF_SUCCESS);
12840	break;
12841	}
12842
12843	case IEMXCPTRAISE_REEXEC_INSTR:
12844	Assert(rcStrict == VINF_SUCCESS);
12845	break;
12846
12847	case IEMXCPTRAISE_DOUBLE_FAULT:
12848	{
12849	/*
12850	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
12851	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
12852	*/
12853	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
12854	{
12855	pVmxTransient->fVectoringDoublePF = true;
12856	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
12857	pVCpu->cpum.GstCtx.cr2));
12858	rcStrict = VINF_SUCCESS;
12859	}
12860	else
12861	{
12862	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
12863	hmR0VmxSetPendingXcptDF(pVCpu);
12864	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
12865	uIdtVector, uExitVector));
12866	rcStrict = VINF_HM_DOUBLE_FAULT;
12867	}
12868	break;
12869	}
12870
12871	case IEMXCPTRAISE_TRIPLE_FAULT:
12872	{
12873	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
12874	rcStrict = VINF_EM_RESET;
12875	break;
12876	}
12877
12878	case IEMXCPTRAISE_CPU_HANG:
12879	{
12880	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
12881	rcStrict = VERR_EM_GUEST_CPU_HANG;
12882	break;
12883	}
12884
12885	default:
12886	{
12887	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
12888	rcStrict = VERR_VMX_IPE_2;
12889	break;
12890	}
12891	}
12892	}
12893	else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
12894	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
12895	&& uExitVector != X86_XCPT_DF
12896	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
12897	{
12898	/*
12899	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
12900	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
12901	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
12902	*/
12903	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
12904	{
12905	Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
12906	VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
12907	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
12908	}
12909	}
12910
12911	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
12912	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
12913	return rcStrict;
12914	}
12915
12916
12917	/** @name VM-exit handlers.
12918	* @{
12919	*/
12920	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12921	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12922	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12923
12924	/**
12925	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
12926	*/
12927	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12928	{
12929	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12930	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
12931	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
12932	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
12933	return VINF_SUCCESS;
12934	return VINF_EM_RAW_INTERRUPT;
12935	}
12936
12937
12938	/**
12939	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
12940	*/
12941	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12942	{
12943	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12944	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
12945
12946	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12947	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12948	AssertRCReturn(rc, rc);
12949
12950	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12951	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
12952	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
12953	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
12954
12955	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12956	{
12957	/*
12958	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
12959	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
12960	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
12961	*
12962	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
12963	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
12964	*/
12965	VMXDispatchHostNmi();
12966	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
12967	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12968	return VINF_SUCCESS;
12969	}
12970
12971	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12972	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12973	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
12974	{ /* likely */ }
12975	else
12976	{
12977	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
12978	rcStrictRc1 = VINF_SUCCESS;
12979	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12980	return rcStrictRc1;
12981	}
12982
12983	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
12984	uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
12985	switch (uIntType)
12986	{
12987	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
12988	Assert(uVector == X86_XCPT_DB);
12989	RT_FALL_THRU();
12990	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
12991	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
12992	RT_FALL_THRU();
12993	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
12994	{
12995	/*
12996	* If there's any exception caused as a result of event injection, the resulting
12997	* secondary/final execption will be pending, we shall continue guest execution
12998	* after injecting the event. The page-fault case is complicated and we manually
12999	* handle any currently pending event in hmR0VmxExitXcptPF.
13000	*/
13001	if (!pVCpu->hm.s.Event.fPending)
13002	{ /* likely */ }
13003	else if (uVector != X86_XCPT_PF)
13004	{
13005	rc = VINF_SUCCESS;
13006	break;
13007	}
13008
13009	switch (uVector)
13010	{
13011	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
13012	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
13013	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
13014	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
13015	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
13016	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
13017
13018	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13019	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13020	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
13021	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13022	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
13023	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13024	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
13025	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13026	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
13027	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13028	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
13029	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13030	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
13031	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13032	default:
13033	{
13034	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
13035	if (pVmcsInfo->RealMode.fRealOnV86Active)
13036	{
13037	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
13038	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
13039	Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
13040
13041	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
13042	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13043	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13044	AssertRCReturn(rc, rc);
13045	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
13046	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
13047	0 /* GCPtrFaultAddress */);
13048	}
13049	else
13050	{
13051	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
13052	pVCpu->hm.s.u32HMError = uVector;
13053	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
13054	}
13055	break;
13056	}
13057	}
13058	break;
13059	}
13060
13061	default:
13062	{
13063	pVCpu->hm.s.u32HMError = uExitIntInfo;
13064	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
13065	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
13066	break;
13067	}
13068	}
13069	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
13070	return rc;
13071	}
13072
13073
13074	/**
13075	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
13076	*/
13077	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13078	{
13079	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13080
13081	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
13082	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13083	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
13084	AssertRCReturn(rc, rc);
13085
13086	/* Evaluate and deliver pending events and resume guest execution. */
13087	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
13088	return VINF_SUCCESS;
13089	}
13090
13091
13092	/**
13093	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
13094	*/
13095	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13096	{
13097	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13098
13099	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13100	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
13101	{
13102	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
13103	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13104	}
13105
13106	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS));
13107
13108	/*
13109	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
13110	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
13111	*/
13112	uint32_t fIntrState;
13113	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13114	AssertRCReturn(rc, rc);
13115	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
13116	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
13117	{
13118	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
13119	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
13120
13121	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
13122	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
13123	AssertRCReturn(rc, rc);
13124	}
13125
13126	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
13127	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
13128	AssertRCReturn(rc, rc);
13129
13130	/* Evaluate and deliver pending events and resume guest execution. */
13131	return VINF_SUCCESS;
13132	}
13133
13134
13135	/**
13136	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
13137	*/
13138	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13139	{
13140	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13141	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13142	}
13143
13144
13145	/**
13146	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
13147	*/
13148	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13149	{
13150	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13151	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13152	}
13153
13154
13155	/**
13156	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
13157	*/
13158	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13159	{
13160	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13161
13162	/*
13163	* Get the state we need and update the exit history entry.
13164	*/
13165	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13166	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13167	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13168	AssertRCReturn(rc, rc);
13169
13170	VBOXSTRICTRC rcStrict;
13171	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
13172	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
13173	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
13174	if (!pExitRec)
13175	{
13176	/*
13177	* Regular CPUID instruction execution.
13178	*/
13179	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
13180	if (rcStrict == VINF_SUCCESS)
13181	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13182	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13183	{
13184	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13185	rcStrict = VINF_SUCCESS;
13186	}
13187	}
13188	else
13189	{
13190	/*
13191	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
13192	*/
13193	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13194	AssertRCReturn(rc2, rc2);
13195
13196	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
13197	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
13198
13199	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
13200	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13201
13202	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
13203	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13204	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
13205	}
13206	return rcStrict;
13207	}
13208
13209
13210	/**
13211	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
13212	*/
13213	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13214	{
13215	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13216
13217	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13218	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
13219	AssertRCReturn(rc, rc);
13220
13221	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
13222	return VINF_EM_RAW_EMULATE_INSTR;
13223
13224	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
13225	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13226	}
13227
13228
13229	/**
13230	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
13231	*/
13232	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13233	{
13234	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13235
13236	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13237	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13238	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13239	AssertRCReturn(rc, rc);
13240
13241	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
13242	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13243	{
13244	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13245	we must reset offsetting on VM-entry. See @bugref{6634}. */
13246	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13247	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13248	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13249	}
13250	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13251	{
13252	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13253	rcStrict = VINF_SUCCESS;
13254	}
13255	return rcStrict;
13256	}
13257
13258
13259	/**
13260	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
13261	*/
13262	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13263	{
13264	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13265
13266	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13267	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
13268	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13269	AssertRCReturn(rc, rc);
13270
13271	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
13272	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13273	{
13274	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13275	we must reset offsetting on VM-reentry. See @bugref{6634}. */
13276	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13277	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13278	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13279	}
13280	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13281	{
13282	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13283	rcStrict = VINF_SUCCESS;
13284	}
13285	return rcStrict;
13286	}
13287
13288
13289	/**
13290	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
13291	*/
13292	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13293	{
13294	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13295
13296	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13297	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
13298	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13299	AssertRCReturn(rc, rc);
13300
13301	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13302	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13303	if (RT_LIKELY(rc == VINF_SUCCESS))
13304	{
13305	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13306	Assert(pVmxTransient->cbInstr == 2);
13307	}
13308	else
13309	{
13310	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
13311	rc = VERR_EM_INTERPRETER;
13312	}
13313	return rc;
13314	}
13315
13316
13317	/**
13318	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
13319	*/
13320	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13321	{
13322	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13323
13324	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
13325	if (EMAreHypercallInstructionsEnabled(pVCpu))
13326	{
13327	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13328	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
13329	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13330	AssertRCReturn(rc, rc);
13331
13332	/* Perform the hypercall. */
13333	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
13334	if (rcStrict == VINF_SUCCESS)
13335	{
13336	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13337	AssertRCReturn(rc, rc);
13338	}
13339	else
13340	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
13341	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
13342	\|\| RT_FAILURE(rcStrict));
13343
13344	/* If the hypercall changes anything other than guest's general-purpose registers,
13345	we would need to reload the guest changed bits here before VM-entry. */
13346	}
13347	else
13348	Log4Func(("Hypercalls not enabled\n"));
13349
13350	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
13351	if (RT_FAILURE(rcStrict))
13352	{
13353	hmR0VmxSetPendingXcptUD(pVCpu);
13354	rcStrict = VINF_SUCCESS;
13355	}
13356
13357	return rcStrict;
13358	}
13359
13360
13361	/**
13362	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
13363	*/
13364	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13365	{
13366	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13367	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
13368
13369	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13370	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13371	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13372	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13373	AssertRCReturn(rc, rc);
13374
13375	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
13376
13377	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
13378	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13379	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13380	{
13381	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13382	rcStrict = VINF_SUCCESS;
13383	}
13384	else
13385	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n", pVmxTransient->uExitQual,
13386	VBOXSTRICTRC_VAL(rcStrict)));
13387	return rcStrict;
13388	}
13389
13390
13391	/**
13392	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
13393	*/
13394	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13395	{
13396	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13397
13398	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13399	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13400	AssertRCReturn(rc, rc);
13401
13402	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13403	rc = EMInterpretMonitor(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13404	if (RT_LIKELY(rc == VINF_SUCCESS))
13405	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13406	else
13407	{
13408	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
13409	rc = VERR_EM_INTERPRETER;
13410	}
13411	return rc;
13412	}
13413
13414
13415	/**
13416	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
13417	*/
13418	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13419	{
13420	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13421
13422	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13423	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13424	AssertRCReturn(rc, rc);
13425
13426	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13427	VBOXSTRICTRC rc2 = EMInterpretMWait(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13428	rc = VBOXSTRICTRC_VAL(rc2);
13429	if (RT_LIKELY( rc == VINF_SUCCESS
13430	\|\| rc == VINF_EM_HALT))
13431	{
13432	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13433	AssertRCReturn(rc3, rc3);
13434
13435	if ( rc == VINF_EM_HALT
13436	&& EMMonitorWaitShouldContinue(pVCpu, pCtx))
13437	rc = VINF_SUCCESS;
13438	}
13439	else
13440	{
13441	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
13442	rc = VERR_EM_INTERPRETER;
13443	}
13444	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
13445	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
13446	return rc;
13447	}
13448
13449
13450	/**
13451	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
13452	*/
13453	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13454	{
13455	/*
13456	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
13457	* mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
13458	* treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
13459	* VMX root operation. If we get here, something funny is going on.
13460	*
13461	* See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
13462	*/
13463	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13464	AssertMsgFailed(("Unexpected RSM VM-exit\n"));
13465	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13466	}
13467
13468
13469	/**
13470	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
13471	*/
13472	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13473	{
13474	/*
13475	* This can only happen if we support dual-monitor treatment of SMI, which can be activated
13476	* by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
13477	* this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
13478	* an SMI. If we get here, something funny is going on.
13479	*
13480	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
13481	* See Intel spec. 25.3 "Other Causes of VM-Exits"
13482	*/
13483	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13484	AssertMsgFailed(("Unexpected SMI VM-exit\n"));
13485	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13486	}
13487
13488
13489	/**
13490	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
13491	*/
13492	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13493	{
13494	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
13495	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13496	AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
13497	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13498	}
13499
13500
13501	/**
13502	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
13503	*/
13504	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13505	{
13506	/*
13507	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
13508	* We don't make use of it as our guests don't have direct access to the host LAPIC.
13509	* See Intel spec. 25.3 "Other Causes of VM-exits".
13510	*/
13511	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13512	AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
13513	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13514	}
13515
13516
13517	/**
13518	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
13519	* VM-exit.
13520	*/
13521	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13522	{
13523	/*
13524	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
13525	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
13526	*
13527	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
13528	* See Intel spec. "23.8 Restrictions on VMX operation".
13529	*/
13530	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13531	return VINF_SUCCESS;
13532	}
13533
13534
13535	/**
13536	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
13537	* VM-exit.
13538	*/
13539	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13540	{
13541	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13542	return VINF_EM_RESET;
13543	}
13544
13545
13546	/**
13547	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
13548	*/
13549	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13550	{
13551	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13552
13553	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13554	AssertRCReturn(rc, rc);
13555
13556	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
13557	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
13558	rc = VINF_SUCCESS;
13559	else
13560	rc = VINF_EM_HALT;
13561
13562	if (rc != VINF_SUCCESS)
13563	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
13564	return rc;
13565	}
13566
13567
13568	/**
13569	* VM-exit handler for instructions that result in a \#UD exception delivered to
13570	* the guest.
13571	*/
13572	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13573	{
13574	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13575	hmR0VmxSetPendingXcptUD(pVCpu);
13576	return VINF_SUCCESS;
13577	}
13578
13579
13580	/**
13581	* VM-exit handler for expiry of the VMX-preemption timer.
13582	*/
13583	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13584	{
13585	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13586
13587	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
13588	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13589
13590	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
13591	PVM pVM = pVCpu->CTX_SUFF(pVM);
13592	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
13593	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
13594	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
13595	}
13596
13597
13598	/**
13599	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
13600	*/
13601	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13602	{
13603	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13604
13605	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13606	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13607	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
13608	AssertRCReturn(rc, rc);
13609
13610	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
13611	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13612	: HM_CHANGED_RAISED_XCPT_MASK);
13613
13614	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13615	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
13616
13617	return rcStrict;
13618	}
13619
13620
13621	/**
13622	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
13623	*/
13624	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13625	{
13626	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13627	/** @todo Use VM-exit instruction information. */
13628	return VERR_EM_INTERPRETER;
13629	}
13630
13631
13632	/**
13633	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
13634	* Error VM-exit.
13635	*/
13636	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13637	{
13638	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13639	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13640	AssertRCReturn(rc, rc);
13641
13642	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo);
13643	if (RT_FAILURE(rc))
13644	return rc;
13645
13646	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
13647	NOREF(uInvalidReason);
13648
13649	#ifdef VBOX_STRICT
13650	uint32_t fIntrState;
13651	RTHCUINTREG uHCReg;
13652	uint64_t u64Val;
13653	uint32_t u32Val;
13654	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
13655	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
13656	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
13657	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13658	AssertRCReturn(rc, rc);
13659
13660	Log4(("uInvalidReason %u\n", uInvalidReason));
13661	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
13662	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
13663	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
13664	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
13665
13666	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
13667	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
13668	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
13669	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
13670	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
13671	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13672	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
13673	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
13674	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
13675	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13676	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
13677	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
13678
13679	hmR0DumpRegs(pVCpu);
13680	#endif
13681
13682	return VERR_VMX_INVALID_GUEST_STATE;
13683	}
13684
13685
13686	/**
13687	* VM-exit handler for VM-entry failure due to an MSR-load
13688	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
13689	*/
13690	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13691	{
13692	AssertMsgFailed(("Unexpected MSR-load exit\n"));
13693	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13694	}
13695
13696
13697	/**
13698	* VM-exit handler for VM-entry failure due to a machine-check event
13699	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
13700	*/
13701	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13702	{
13703	AssertMsgFailed(("Unexpected machine-check event exit\n"));
13704	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13705	}
13706
13707
13708	/**
13709	* VM-exit handler for all undefined reasons. Should never ever happen.. in
13710	* theory.
13711	*/
13712	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13713	{
13714	RT_NOREF2(pVCpu, pVmxTransient);
13715	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
13716	return VERR_VMX_UNDEFINED_EXIT_CODE;
13717	}
13718
13719
13720	/**
13721	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
13722	* (VMX_EXIT_GDTR_IDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
13723	* Conditional VM-exit.
13724	*/
13725	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13726	{
13727	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13728
13729	/* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
13730	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
13731	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13732	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
13733	return VERR_EM_INTERPRETER;
13734	AssertMsgFailed(("Unexpected XDTR access\n"));
13735	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13736	}
13737
13738
13739	/**
13740	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
13741	*/
13742	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13743	{
13744	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13745
13746	/* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
13747	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13748	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
13749	return VERR_EM_INTERPRETER;
13750	AssertMsgFailed(("Unexpected RDRAND exit\n"));
13751	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13752	}
13753
13754
13755	/**
13756	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
13757	*/
13758	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13759	{
13760	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13761
13762	/** @todo Optimize this: We currently drag in in the whole MSR state
13763	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13764	* MSRs required. That would require changes to IEM and possibly CPUM too.
13765	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13766	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13767	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13768	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13769	switch (idMsr)
13770	{
13771	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13772	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13773	}
13774
13775	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13776	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13777	AssertRCReturn(rc, rc);
13778
13779	Log4Func(("ecx=%#RX32\n", idMsr));
13780
13781	#ifdef VBOX_STRICT
13782	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
13783	{
13784	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
13785	&& idMsr != MSR_K6_EFER)
13786	{
13787	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
13788	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13789	}
13790	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13791	{
13792	Assert(pVmcsInfo->pvMsrBitmap);
13793	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13794	if (fMsrpm & VMXMSRPM_ALLOW_RD)
13795	{
13796	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
13797	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13798	}
13799	}
13800	}
13801	#endif
13802
13803	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
13804	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
13805	if (rcStrict == VINF_SUCCESS)
13806	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13807	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
13808	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13809	{
13810	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13811	rcStrict = VINF_SUCCESS;
13812	}
13813	else
13814	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13815
13816	return rcStrict;
13817	}
13818
13819
13820	/**
13821	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
13822	*/
13823	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13824	{
13825	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13826
13827	/** @todo Optimize this: We currently drag in in the whole MSR state
13828	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13829	* MSRs required. That would require changes to IEM and possibly CPUM too.
13830	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13831	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13832	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13833
13834	/*
13835	* The FS and GS base MSRs are not part of the above all-MSRs mask.
13836	* Although we don't need to fetch the base as it will be overwritten shortly, while
13837	* loading guest-state we would also load the entire segment register including limit
13838	* and attributes and thus we need to load them here.
13839	*/
13840	switch (idMsr)
13841	{
13842	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13843	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13844	}
13845
13846	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13847	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13848	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13849	AssertRCReturn(rc, rc);
13850
13851	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
13852
13853	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
13854	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
13855
13856	if (rcStrict == VINF_SUCCESS)
13857	{
13858	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13859
13860	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
13861	if ( idMsr == MSR_IA32_APICBASE
13862	\|\| ( idMsr >= MSR_IA32_X2APIC_START
13863	&& idMsr <= MSR_IA32_X2APIC_END))
13864	{
13865	/*
13866	* We've already saved the APIC related guest-state (TPR) in post-run phase.
13867	* When full APIC register virtualization is implemented we'll have to make
13868	* sure APIC state is saved from the VMCS before IEM changes it.
13869	*/
13870	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
13871	}
13872	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
13873	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13874	else if (idMsr == MSR_K6_EFER)
13875	{
13876	/*
13877	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
13878	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
13879	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
13880	*/
13881	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
13882	}
13883
13884	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
13885	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
13886	{
13887	switch (idMsr)
13888	{
13889	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
13890	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
13891	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
13892	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
13893	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
13894	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
13895	default:
13896	{
13897	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
13898	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
13899	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13900	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
13901	break;
13902	}
13903	}
13904	}
13905	#ifdef VBOX_STRICT
13906	else
13907	{
13908	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
13909	switch (idMsr)
13910	{
13911	case MSR_IA32_SYSENTER_CS:
13912	case MSR_IA32_SYSENTER_EIP:
13913	case MSR_IA32_SYSENTER_ESP:
13914	case MSR_K8_FS_BASE:
13915	case MSR_K8_GS_BASE:
13916	{
13917	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
13918	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13919	}
13920
13921	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
13922	default:
13923	{
13924	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
13925	{
13926	/* EFER MSR writes are always intercepted. */
13927	if (idMsr != MSR_K6_EFER)
13928	{
13929	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
13930	idMsr));
13931	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13932	}
13933	}
13934
13935	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13936	{
13937	Assert(pVmcsInfo->pvMsrBitmap);
13938	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13939	if (fMsrpm & VMXMSRPM_ALLOW_WR)
13940	{
13941	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
13942	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13943	}
13944	}
13945	break;
13946	}
13947	}
13948	}
13949	#endif /* VBOX_STRICT */
13950	}
13951	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13952	{
13953	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13954	rcStrict = VINF_SUCCESS;
13955	}
13956	else
13957	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13958
13959	return rcStrict;
13960	}
13961
13962
13963	/**
13964	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
13965	*/
13966	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13967	{
13968	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13969	/** @todo The guest has likely hit a contended spinlock. We might want to
13970	* poke a schedule different guest VCPU. */
13971	return VINF_EM_RAW_INTERRUPT;
13972	}
13973
13974
13975	/**
13976	* VM-exit handler for when the TPR value is lowered below the specified
13977	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
13978	*/
13979	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13980	{
13981	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13982	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
13983
13984	/*
13985	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
13986	* We'll re-evaluate pending interrupts and inject them before the next VM
13987	* entry so we can just continue execution here.
13988	*/
13989	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
13990	return VINF_SUCCESS;
13991	}
13992
13993
13994	/**
13995	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
13996	* VM-exit.
13997	*
13998	* @retval VINF_SUCCESS when guest execution can continue.
13999	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
14000	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
14001	* interpreter.
14002	*/
14003	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14004	{
14005	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14006	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
14007
14008	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14009	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14010	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14011	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14012	AssertRCReturn(rc, rc);
14013
14014	VBOXSTRICTRC rcStrict;
14015	PVM pVM = pVCpu->CTX_SUFF(pVM);
14016	RTGCUINTPTR const uExitQual = pVmxTransient->uExitQual;
14017	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
14018	switch (uAccessType)
14019	{
14020	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
14021	{
14022	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
14023	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
14024	VMX_EXIT_QUAL_CRX_GENREG(uExitQual));
14025	AssertMsg( rcStrict == VINF_SUCCESS
14026	\|\| rcStrict == VINF_IEM_RAISED_XCPT
14027	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14028
14029	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
14030	{
14031	case 0:
14032	{
14033	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14034	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14035	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
14036	Log4Func(("CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
14037
14038	/*
14039	* This is a kludge for handling switches back to real mode when we try to use
14040	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
14041	* deal with special selector values, so we have to return to ring-3 and run
14042	* there till the selector values are V86 mode compatible.
14043	*
14044	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
14045	* latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
14046	* at the end of this function.
14047	*/
14048	if ( rc == VINF_SUCCESS
14049	&& !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
14050	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
14051	&& (uOldCr0 & X86_CR0_PE)
14052	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
14053	{
14054	/** @todo check selectors rather than returning all the time. */
14055	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
14056	rcStrict = VINF_EM_RESCHEDULE_REM;
14057	}
14058	break;
14059	}
14060
14061	case 2:
14062	{
14063	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
14064	/* Nothing to do here, CR2 it's not part of the VMCS. */
14065	break;
14066	}
14067
14068	case 3:
14069	{
14070	Assert( !pVM->hm.s.fNestedPaging
14071	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
14072	\|\| pVCpu->hm.s.fUsingDebugLoop);
14073	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
14074	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14075	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
14076	Log4Func(("CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
14077	break;
14078	}
14079
14080	case 4:
14081	{
14082	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
14083	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14084	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
14085	Log4Func(("CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
14086	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
14087	break;
14088	}
14089
14090	case 8:
14091	{
14092	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
14093	Assert(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
14094	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14095	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
14096	break;
14097	}
14098	default:
14099	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual)));
14100	break;
14101	}
14102	break;
14103	}
14104
14105	case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
14106	{
14107	Assert( !pVM->hm.s.fNestedPaging
14108	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
14109	\|\| pVCpu->hm.s.fUsingDebugLoop
14110	\|\| VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 3);
14111	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
14112	Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 8
14113	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
14114
14115	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_GENREG(uExitQual),
14116	VMX_EXIT_QUAL_CRX_REGISTER(uExitQual));
14117	AssertMsg( rcStrict == VINF_SUCCESS
14118	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14119	#ifdef VBOX_WITH_STATISTICS
14120	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
14121	{
14122	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
14123	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
14124	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
14125	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
14126	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
14127	}
14128	#endif
14129	Log4Func(("CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
14130	VBOXSTRICTRC_VAL(rcStrict)));
14131	if (VMX_EXIT_QUAL_CRX_GENREG(uExitQual) == X86_GREG_xSP)
14132	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
14133	else
14134	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14135	break;
14136	}
14137
14138	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
14139	{
14140	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
14141	AssertMsg( rcStrict == VINF_SUCCESS
14142	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14143
14144	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14145	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
14146	Log4Func(("CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
14147	break;
14148	}
14149
14150	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
14151	{
14152	/* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
14153	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
14154	AssertRCReturn(rc, rc);
14155	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual),
14156	pVmxTransient->uGuestLinearAddr);
14157	AssertMsg( rcStrict == VINF_SUCCESS
14158	\|\| rcStrict == VINF_IEM_RAISED_XCPT
14159	, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14160
14161	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14162	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
14163	Log4Func(("LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
14164	break;
14165	}
14166
14167	default:
14168	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
14169	VERR_VMX_UNEXPECTED_EXCEPTION);
14170	}
14171
14172	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
14173	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
14174	if (rcStrict == VINF_IEM_RAISED_XCPT)
14175	{
14176	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14177	rcStrict = VINF_SUCCESS;
14178	}
14179
14180	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
14181	NOREF(pVM);
14182	return rcStrict;
14183	}
14184
14185
14186	/**
14187	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
14188	* VM-exit.
14189	*/
14190	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14191	{
14192	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14193	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
14194
14195	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14196	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14197	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14198	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14199	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
14200	\| CPUMCTX_EXTRN_EFER);
14201	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
14202	AssertRCReturn(rc, rc);
14203
14204	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
14205	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
14206	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
14207	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
14208	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
14209	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
14210	bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
14211	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
14212
14213	/*
14214	* Update exit history to see if this exit can be optimized.
14215	*/
14216	VBOXSTRICTRC rcStrict;
14217	PCEMEXITREC pExitRec = NULL;
14218	if ( !fGstStepping
14219	&& !fDbgStepping)
14220	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14221	!fIOString
14222	? !fIOWrite
14223	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
14224	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
14225	: !fIOWrite
14226	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
14227	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
14228	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14229	if (!pExitRec)
14230	{
14231	/* I/O operation lookup arrays. */
14232	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
14233	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
14234	uint32_t const cbValue = s_aIOSizes[uIOSize];
14235	uint32_t const cbInstr = pVmxTransient->cbInstr;
14236	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
14237	PVM pVM = pVCpu->CTX_SUFF(pVM);
14238	if (fIOString)
14239	{
14240	/*
14241	* INS/OUTS - I/O String instruction.
14242	*
14243	* Use instruction-information if available, otherwise fall back on
14244	* interpreting the instruction.
14245	*/
14246	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14247	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
14248	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
14249	if (fInsOutsInfo)
14250	{
14251	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14252	AssertRCReturn(rc2, rc2);
14253	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
14254	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
14255	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
14256	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
14257	if (fIOWrite)
14258	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
14259	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
14260	else
14261	{
14262	/*
14263	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
14264	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
14265	* See Intel Instruction spec. for "INS".
14266	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
14267	*/
14268	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
14269	}
14270	}
14271	else
14272	rcStrict = IEMExecOne(pVCpu);
14273
14274	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14275	fUpdateRipAlready = true;
14276	}
14277	else
14278	{
14279	/*
14280	* IN/OUT - I/O instruction.
14281	*/
14282	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14283	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
14284	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
14285	if (fIOWrite)
14286	{
14287	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
14288	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
14289	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14290	&& !pCtx->eflags.Bits.u1TF)
14291	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
14292	}
14293	else
14294	{
14295	uint32_t u32Result = 0;
14296	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
14297	if (IOM_SUCCESS(rcStrict))
14298	{
14299	/* Save result of I/O IN instr. in AL/AX/EAX. */
14300	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
14301	}
14302	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14303	&& !pCtx->eflags.Bits.u1TF)
14304	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
14305	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
14306	}
14307	}
14308
14309	if (IOM_SUCCESS(rcStrict))
14310	{
14311	if (!fUpdateRipAlready)
14312	{
14313	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
14314	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14315	}
14316
14317	/*
14318	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
14319	* while booting Fedora 17 64-bit guest.
14320	*
14321	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
14322	*/
14323	if (fIOString)
14324	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
14325
14326	/*
14327	* If any I/O breakpoints are armed, we need to check if one triggered
14328	* and take appropriate action.
14329	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
14330	*/
14331	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
14332	AssertRCReturn(rc, rc);
14333
14334	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
14335	* execution engines about whether hyper BPs and such are pending. */
14336	uint32_t const uDr7 = pCtx->dr[7];
14337	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
14338	&& X86_DR7_ANY_RW_IO(uDr7)
14339	&& (pCtx->cr4 & X86_CR4_DE))
14340	\|\| DBGFBpIsHwIoArmed(pVM)))
14341	{
14342	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
14343
14344	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
14345	VMMRZCallRing3Disable(pVCpu);
14346	HM_DISABLE_PREEMPT(pVCpu);
14347
14348	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
14349
14350	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
14351	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
14352	{
14353	/* Raise #DB. */
14354	if (fIsGuestDbgActive)
14355	ASMSetDR6(pCtx->dr[6]);
14356	if (pCtx->dr[7] != uDr7)
14357	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
14358
14359	hmR0VmxSetPendingXcptDB(pVCpu);
14360	}
14361	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
14362	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
14363	else if ( rcStrict2 != VINF_SUCCESS
14364	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
14365	rcStrict = rcStrict2;
14366	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
14367
14368	HM_RESTORE_PREEMPT();
14369	VMMRZCallRing3Enable(pVCpu);
14370	}
14371	}
14372
14373	#ifdef VBOX_STRICT
14374	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14375	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
14376	Assert(!fIOWrite);
14377	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14378	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
14379	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
14380	Assert(fIOWrite);
14381	else
14382	{
14383	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
14384	* statuses, that the VMM device and some others may return. See
14385	* IOM_SUCCESS() for guidance. */
14386	AssertMsg( RT_FAILURE(rcStrict)
14387	\|\| rcStrict == VINF_SUCCESS
14388	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
14389	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
14390	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
14391	\|\| rcStrict == VINF_EM_RAW_TO_R3
14392	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14393	# endif
14394	}
14395	#endif
14396	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
14397	}
14398	else
14399	{
14400	/*
14401	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14402	*/
14403	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14404	AssertRCReturn(rc2, rc2);
14405	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
14406	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
14407	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
14408	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14409	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
14410	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
14411
14412	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14413	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14414
14415	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14416	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14417	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14418	}
14419	return rcStrict;
14420	}
14421
14422
14423	/**
14424	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
14425	* VM-exit.
14426	*/
14427	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14428	{
14429	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14430
14431	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
14432	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14433	AssertRCReturn(rc, rc);
14434	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
14435	{
14436	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14437	AssertRCReturn(rc, rc);
14438	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
14439	{
14440	uint32_t uErrCode;
14441	RTGCUINTPTR GCPtrFaultAddress;
14442	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
14443	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
14444	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
14445	if (fErrorCodeValid)
14446	{
14447	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14448	AssertRCReturn(rc, rc);
14449	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
14450	}
14451	else
14452	uErrCode = 0;
14453
14454	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
14455	&& uVector == X86_XCPT_PF)
14456	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
14457	else
14458	GCPtrFaultAddress = 0;
14459
14460	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14461	AssertRCReturn(rc, rc);
14462
14463	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
14464	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
14465
14466	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
14467	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14468	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14469	}
14470	}
14471
14472	/* Fall back to the interpreter to emulate the task-switch. */
14473	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14474	return VERR_EM_INTERPRETER;
14475	}
14476
14477
14478	/**
14479	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
14480	*/
14481	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14482	{
14483	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14484
14485	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14486	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
14487	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14488	AssertRCReturn(rc, rc);
14489	return VINF_EM_DBG_STEPPED;
14490	}
14491
14492
14493	/**
14494	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
14495	*/
14496	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14497	{
14498	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14499	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
14500
14501	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14502	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14503	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14504	{
14505	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
14506	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14507	{
14508	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14509	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14510	}
14511	}
14512	else
14513	{
14514	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14515	rcStrict1 = VINF_SUCCESS;
14516	return rcStrict1;
14517	}
14518
14519	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
14520	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14521	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14522	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14523	AssertRCReturn(rc, rc);
14524
14525	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
14526	uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
14527	VBOXSTRICTRC rcStrict2;
14528	switch (uAccessType)
14529	{
14530	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
14531	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
14532	{
14533	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
14534	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
14535	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
14536
14537	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
14538	GCPhys &= PAGE_BASE_GC_MASK;
14539	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
14540	PVM pVM = pVCpu->CTX_SUFF(pVM);
14541	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
14542	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
14543
14544	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14545	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
14546	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
14547	CPUMCTX2CORE(pCtx), GCPhys);
14548	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14549	if ( rcStrict2 == VINF_SUCCESS
14550	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14551	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14552	{
14553	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14554	\| HM_CHANGED_GUEST_APIC_TPR);
14555	rcStrict2 = VINF_SUCCESS;
14556	}
14557	break;
14558	}
14559
14560	default:
14561	Log4Func(("uAccessType=%#x\n", uAccessType));
14562	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
14563	break;
14564	}
14565
14566	if (rcStrict2 != VINF_SUCCESS)
14567	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
14568	return rcStrict2;
14569	}
14570
14571
14572	/**
14573	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
14574	* VM-exit.
14575	*/
14576	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14577	{
14578	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14579
14580	/* We should -not- get this VM-exit if the guest's debug registers were active. */
14581	if (pVmxTransient->fWasGuestDebugStateActive)
14582	{
14583	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
14584	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14585	}
14586
14587	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14588	if ( !pVCpu->hm.s.fSingleInstruction
14589	&& !pVmxTransient->fWasHyperDebugStateActive)
14590	{
14591	Assert(!DBGFIsStepping(pVCpu));
14592	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
14593
14594	/* Don't intercept MOV DRx any more. */
14595	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
14596	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14597	AssertRCReturn(rc, rc);
14598
14599	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
14600	VMMRZCallRing3Disable(pVCpu);
14601	HM_DISABLE_PREEMPT(pVCpu);
14602
14603	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
14604	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
14605	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
14606
14607	HM_RESTORE_PREEMPT();
14608	VMMRZCallRing3Enable(pVCpu);
14609
14610	#ifdef VBOX_WITH_STATISTICS
14611	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14612	AssertRCReturn(rc, rc);
14613	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14614	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14615	else
14616	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14617	#endif
14618	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
14619	return VINF_SUCCESS;
14620	}
14621
14622	/*
14623	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
14624	* The EFER MSR is always up-to-date.
14625	* Update the segment registers and DR7 from the CPU.
14626	*/
14627	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14628	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14629	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
14630	AssertRCReturn(rc, rc);
14631	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
14632
14633	PVM pVM = pVCpu->CTX_SUFF(pVM);
14634	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14635	{
14636	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14637	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
14638	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
14639	if (RT_SUCCESS(rc))
14640	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
14641	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14642	}
14643	else
14644	{
14645	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14646	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
14647	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
14648	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14649	}
14650
14651	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
14652	if (RT_SUCCESS(rc))
14653	{
14654	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14655	AssertRCReturn(rc2, rc2);
14656	return VINF_SUCCESS;
14657	}
14658	return rc;
14659	}
14660
14661
14662	/**
14663	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
14664	* Conditional VM-exit.
14665	*/
14666	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14667	{
14668	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14669	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14670
14671	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14672	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14673	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14674	{
14675	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
14676	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
14677	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14678	{
14679	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14680	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14681	}
14682	}
14683	else
14684	{
14685	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14686	rcStrict1 = VINF_SUCCESS;
14687	return rcStrict1;
14688	}
14689
14690	/*
14691	* Get sufficent state and update the exit history entry.
14692	*/
14693	RTGCPHYS GCPhys;
14694	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14695	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14696	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14697	AssertRCReturn(rc, rc);
14698
14699	VBOXSTRICTRC rcStrict;
14700	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14701	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
14702	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14703	if (!pExitRec)
14704	{
14705	/*
14706	* If we succeed, resume guest execution.
14707	* If we fail in interpreting the instruction because we couldn't get the guest physical address
14708	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
14709	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
14710	* weird case. See @bugref{6043}.
14711	*/
14712	PVM pVM = pVCpu->CTX_SUFF(pVM);
14713	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14714	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
14715	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
14716	if ( rcStrict == VINF_SUCCESS
14717	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
14718	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
14719	{
14720	/* Successfully handled MMIO operation. */
14721	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14722	\| HM_CHANGED_GUEST_APIC_TPR);
14723	rcStrict = VINF_SUCCESS;
14724	}
14725	}
14726	else
14727	{
14728	/*
14729	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14730	*/
14731	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14732	AssertRCReturn(rc2, rc2);
14733
14734	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
14735	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
14736
14737	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14738	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14739
14740	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14741	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14742	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14743	}
14744	return VBOXSTRICTRC_TODO(rcStrict);
14745	}
14746
14747
14748	/**
14749	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
14750	* VM-exit.
14751	*/
14752	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14753	{
14754	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14755	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14756
14757	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14758	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14759	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14760	{
14761	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
14762	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14763	Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
14764	}
14765	else
14766	{
14767	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14768	rcStrict1 = VINF_SUCCESS;
14769	return rcStrict1;
14770	}
14771
14772	RTGCPHYS GCPhys;
14773	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14774	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14775	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14776	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14777	AssertRCReturn(rc, rc);
14778
14779	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
14780	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
14781
14782	RTGCUINT uErrorCode = 0;
14783	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
14784	uErrorCode \|= X86_TRAP_PF_ID;
14785	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
14786	uErrorCode \|= X86_TRAP_PF_RW;
14787	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
14788	uErrorCode \|= X86_TRAP_PF_P;
14789
14790	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
14791
14792
14793	/* Handle the pagefault trap for the nested shadow table. */
14794	PVM pVM = pVCpu->CTX_SUFF(pVM);
14795	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14796
14797	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
14798	pCtx->cs.Sel, pCtx->rip));
14799
14800	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
14801	TRPMResetTrap(pVCpu);
14802
14803	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
14804	if ( rcStrict2 == VINF_SUCCESS
14805	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14806	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14807	{
14808	/* Successfully synced our nested page tables. */
14809	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
14810	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
14811	return VINF_SUCCESS;
14812	}
14813
14814	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14815	return rcStrict2;
14816	}
14817
14818	/** @} */
14819
14820	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14821	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14822	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14823
14824	/**
14825	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
14826	*/
14827	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14828	{
14829	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14830	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
14831
14832	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
14833	AssertRCReturn(rc, rc);
14834
14835	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
14836	{
14837	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
14838	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
14839
14840	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
14841	* provides VM-exit instruction length. If this causes problem later,
14842	* disassemble the instruction like it's done on AMD-V. */
14843	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14844	AssertRCReturn(rc2, rc2);
14845	return rc;
14846	}
14847
14848	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14849	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14850	return rc;
14851	}
14852
14853
14854	/**
14855	* VM-exit exception handler for \#BP (Breakpoint exception).
14856	*/
14857	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14858	{
14859	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14860	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
14861
14862	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14863	AssertRCReturn(rc, rc);
14864
14865	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14866	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14867	if (rc == VINF_EM_RAW_GUEST_TRAP)
14868	{
14869	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14870	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14871	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14872	AssertRCReturn(rc, rc);
14873
14874	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14875	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14876	}
14877
14878	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
14879	return rc;
14880	}
14881
14882
14883	/**
14884	* VM-exit exception handler for \#AC (alignment check exception).
14885	*/
14886	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14887	{
14888	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14889
14890	/*
14891	* Re-inject it. We'll detect any nesting before getting here.
14892	*/
14893	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14894	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14895	AssertRCReturn(rc, rc);
14896	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
14897
14898	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14899	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14900	return VINF_SUCCESS;
14901	}
14902
14903
14904	/**
14905	* VM-exit exception handler for \#DB (Debug exception).
14906	*/
14907	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14908	{
14909	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14910	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
14911
14912	/*
14913	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
14914	* for processing.
14915	*/
14916	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14917
14918	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
14919	uint64_t const uDR6 = X86_DR6_INIT_VAL
14920	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
14921	\| X86_DR6_BD \| X86_DR6_BS));
14922
14923	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14924	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
14925	Log6Func(("rc=%Rrc\n", rc));
14926	if (rc == VINF_EM_RAW_GUEST_TRAP)
14927	{
14928	/*
14929	* The exception was for the guest. Update DR6, DR7.GD and
14930	* IA32_DEBUGCTL.LBR before forwarding it.
14931	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
14932	*/
14933	VMMRZCallRing3Disable(pVCpu);
14934	HM_DISABLE_PREEMPT(pVCpu);
14935
14936	pCtx->dr[6] &= ~X86_DR6_B_MASK;
14937	pCtx->dr[6] \|= uDR6;
14938	if (CPUMIsGuestDebugStateActive(pVCpu))
14939	ASMSetDR6(pCtx->dr[6]);
14940
14941	HM_RESTORE_PREEMPT();
14942	VMMRZCallRing3Enable(pVCpu);
14943
14944	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
14945	AssertRCReturn(rc, rc);
14946
14947	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
14948	pCtx->dr[7] &= ~X86_DR7_GD;
14949
14950	/* Paranoia. */
14951	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
14952	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
14953
14954	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
14955	AssertRCReturn(rc, rc);
14956
14957	/*
14958	* Raise #DB in the guest.
14959	*
14960	* It is important to reflect exactly what the VM-exit gave us (preserving the
14961	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
14962	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
14963	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
14964	*
14965	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
14966	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
14967	*/
14968	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14969	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14970	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14971	AssertRCReturn(rc, rc);
14972	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14973	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14974	return VINF_SUCCESS;
14975	}
14976
14977	/*
14978	* Not a guest trap, must be a hypervisor related debug event then.
14979	* Update DR6 in case someone is interested in it.
14980	*/
14981	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
14982	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
14983	CPUMSetHyperDR6(pVCpu, uDR6);
14984
14985	return rc;
14986	}
14987
14988
14989	/**
14990	* Hacks its way around the lovely mesa driver's backdoor accesses.
14991	*
14992	* @sa hmR0SvmHandleMesaDrvGp.
14993	*/
14994	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14995	{
14996	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
14997	RT_NOREF(pCtx);
14998
14999	/* For now we'll just skip the instruction. */
15000	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15001	}
15002
15003
15004	/**
15005	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
15006	* backdoor logging w/o checking what it is running inside.
15007	*
15008	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
15009	* backdoor port and magic numbers loaded in registers.
15010	*
15011	* @returns true if it is, false if it isn't.
15012	* @sa hmR0SvmIsMesaDrvGp.
15013	*/
15014	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
15015	{
15016	/* 0xed: IN eAX,dx */
15017	uint8_t abInstr[1];
15018	if (pVmxTransient->cbInstr != sizeof(abInstr))
15019	return false;
15020
15021	/* Check that it is #GP(0). */
15022	if (pVmxTransient->uExitIntErrorCode != 0)
15023	return false;
15024
15025	/* Check magic and port. */
15026	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
15027	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
15028	if (pCtx->rax != UINT32_C(0x564d5868))
15029	return false;
15030	if (pCtx->dx != UINT32_C(0x5658))
15031	return false;
15032
15033	/* Flat ring-3 CS. */
15034	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
15035	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
15036	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
15037	if (pCtx->cs.Attr.n.u2Dpl != 3)
15038	return false;
15039	if (pCtx->cs.u64Base != 0)
15040	return false;
15041
15042	/* Check opcode. */
15043	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
15044	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
15045	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
15046	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
15047	if (RT_FAILURE(rc))
15048	return false;
15049	if (abInstr[0] != 0xed)
15050	return false;
15051
15052	return true;
15053	}
15054
15055
15056	/**
15057	* VM-exit exception handler for \#GP (General-protection exception).
15058	*
15059	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
15060	*/
15061	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15062	{
15063	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15064	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
15065
15066	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15067	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15068	if (pVmcsInfo->RealMode.fRealOnV86Active)
15069	{ /* likely */ }
15070	else
15071	{
15072	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
15073	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv);
15074	#endif
15075	/* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
15076	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15077	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15078	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15079	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15080	AssertRCReturn(rc, rc);
15081	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
15082	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
15083
15084	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
15085	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
15086	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
15087	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15088	else
15089	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
15090	return rc;
15091	}
15092
15093	Assert(CPUMIsGuestInRealModeEx(pCtx));
15094	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
15095
15096	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15097	AssertRCReturn(rc, rc);
15098
15099	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
15100	if (rcStrict == VINF_SUCCESS)
15101	{
15102	if (!CPUMIsGuestInRealModeEx(pCtx))
15103	{
15104	/*
15105	* The guest is no longer in real-mode, check if we can continue executing the
15106	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
15107	*/
15108	pVmcsInfo->RealMode.fRealOnV86Active = false;
15109	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
15110	{
15111	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
15112	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15113	}
15114	else
15115	{
15116	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
15117	rcStrict = VINF_EM_RESCHEDULE;
15118	}
15119	}
15120	else
15121	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15122	}
15123	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15124	{
15125	rcStrict = VINF_SUCCESS;
15126	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15127	}
15128	return VBOXSTRICTRC_VAL(rcStrict);
15129	}
15130
15131
15132	/**
15133	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
15134	* the exception reported in the VMX transient structure back into the VM.
15135	*
15136	* @remarks Requires uExitIntInfo in the VMX transient structure to be
15137	* up-to-date.
15138	*/
15139	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15140	{
15141	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15142	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
15143	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15144	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active,
15145	("uVector=%#x u32XcptBitmap=%#X32\n",
15146	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
15147	NOREF(pVmcsInfo);
15148	#endif
15149
15150	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
15151	hmR0VmxCheckExitDueToEventDelivery(). */
15152	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15153	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15154	AssertRCReturn(rc, rc);
15155	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15156
15157	#ifdef DEBUG_ramshankar
15158	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
15159	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n",
15160	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pCtx->cs.Sel, pCtx->rip));
15161	#endif
15162
15163	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15164	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15165	return VINF_SUCCESS;
15166	}
15167
15168
15169	/**
15170	* VM-exit exception handler for \#PF (Page-fault exception).
15171	*/
15172	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15173	{
15174	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15175	PVM pVM = pVCpu->CTX_SUFF(pVM);
15176	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15177	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15178	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15179	AssertRCReturn(rc, rc);
15180
15181	if (!pVM->hm.s.fNestedPaging)
15182	{ /* likely */ }
15183	else
15184	{
15185	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
15186	Assert(pVCpu->hm.s.fUsingDebugLoop);
15187	#endif
15188	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
15189	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
15190	{
15191	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15192	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
15193	}
15194	else
15195	{
15196	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15197	hmR0VmxSetPendingXcptDF(pVCpu);
15198	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
15199	}
15200	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15201	return rc;
15202	}
15203
15204	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
15205	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
15206	if (pVmxTransient->fVectoringPF)
15207	{
15208	Assert(pVCpu->hm.s.Event.fPending);
15209	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15210	}
15211
15212	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15213	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15214	AssertRCReturn(rc, rc);
15215
15216	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
15217	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
15218
15219	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
15220	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
15221
15222	Log4Func(("#PF: rc=%Rrc\n", rc));
15223	if (rc == VINF_SUCCESS)
15224	{
15225	/*
15226	* This is typically a shadow page table sync or a MMIO instruction. But we may have
15227	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
15228	*/
15229	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15230	TRPMResetTrap(pVCpu);
15231	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
15232	return rc;
15233	}
15234
15235	if (rc == VINF_EM_RAW_GUEST_TRAP)
15236	{
15237	if (!pVmxTransient->fVectoringDoublePF)
15238	{
15239	/* It's a guest page fault and needs to be reflected to the guest. */
15240	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
15241	TRPMResetTrap(pVCpu);
15242	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
15243	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15244	uGstErrorCode, pVmxTransient->uExitQual);
15245	}
15246	else
15247	{
15248	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15249	TRPMResetTrap(pVCpu);
15250	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
15251	hmR0VmxSetPendingXcptDF(pVCpu);
15252	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
15253	}
15254
15255	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15256	return VINF_SUCCESS;
15257	}
15258
15259	TRPMResetTrap(pVCpu);
15260	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
15261	return rc;
15262	}
15263
15264	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15265	/** @name VMX instruction handlers.
15266	* @{
15267	*/
15268	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15269	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VMX instructions VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15270	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15271
15272	/**
15273	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
15274	*/
15275	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15276	{
15277	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15278
15279	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15280	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15281	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15282	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15283	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15284	AssertRCReturn(rc, rc);
15285
15286	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15287
15288	VMXVEXITINFO ExitInfo;
15289	RT_ZERO(ExitInfo);
15290	ExitInfo.uReason = pVmxTransient->uExitReason;
15291	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15292	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15293	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15294	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15295
15296	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
15297	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15298	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15299	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15300	{
15301	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15302	rcStrict = VINF_SUCCESS;
15303	}
15304	return rcStrict;
15305	}
15306
15307
15308	/**
15309	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
15310	*/
15311	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15312	{
15313	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15314
15315	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
15316	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15317	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15318	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15319	AssertRCReturn(rc, rc);
15320
15321	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15322
15323	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
15324	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15325	{
15326	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15327	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15328	}
15329	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15330	return rcStrict;
15331	}
15332
15333
15334	/**
15335	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
15336	*/
15337	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15338	{
15339	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15340
15341	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15342	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15343	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15344	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15345	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15346	AssertRCReturn(rc, rc);
15347
15348	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15349
15350	VMXVEXITINFO ExitInfo;
15351	RT_ZERO(ExitInfo);
15352	ExitInfo.uReason = pVmxTransient->uExitReason;
15353	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15354	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15355	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15356	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15357
15358	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
15359	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15360	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15361	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15362	{
15363	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15364	rcStrict = VINF_SUCCESS;
15365	}
15366	return rcStrict;
15367	}
15368
15369
15370	/**
15371	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
15372	*/
15373	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15374	{
15375	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15376
15377	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15378	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15379	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15380	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15381	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15382	AssertRCReturn(rc, rc);
15383
15384	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15385
15386	VMXVEXITINFO ExitInfo;
15387	RT_ZERO(ExitInfo);
15388	ExitInfo.uReason = pVmxTransient->uExitReason;
15389	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15390	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15391	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15392	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15393
15394	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
15395	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15396	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15397	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15398	{
15399	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15400	rcStrict = VINF_SUCCESS;
15401	}
15402	return rcStrict;
15403	}
15404
15405
15406	/**
15407	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Unconditional VM-exit.
15408	*/
15409	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15410	{
15411	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15412
15413	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15414	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15415	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15416	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15417	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15418	AssertRCReturn(rc, rc);
15419
15420	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15421
15422	VMXVEXITINFO ExitInfo;
15423	RT_ZERO(ExitInfo);
15424	ExitInfo.uReason = pVmxTransient->uExitReason;
15425	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15426	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15427	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15428	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15429	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15430
15431	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
15432	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15433	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15434	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15435	{
15436	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15437	rcStrict = VINF_SUCCESS;
15438	}
15439	return rcStrict;
15440	}
15441
15442
15443	/**
15444	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
15445	*/
15446	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15447	{
15448	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15449
15450	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
15451	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15452	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15453	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15454	AssertRCReturn(rc, rc);
15455
15456	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15457
15458	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
15459	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15460	{
15461	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15462	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15463	}
15464	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15465	return rcStrict;
15466	}
15467
15468
15469	/**
15470	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Unconditional VM-exit.
15471	*/
15472	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15473	{
15474	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15475
15476	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15477	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15478	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15479	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15480	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15481	AssertRCReturn(rc, rc);
15482
15483	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15484
15485	VMXVEXITINFO ExitInfo;
15486	RT_ZERO(ExitInfo);
15487	ExitInfo.uReason = pVmxTransient->uExitReason;
15488	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15489	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15490	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15491	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15492	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15493
15494	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
15495	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15496	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15497	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15498	{
15499	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15500	rcStrict = VINF_SUCCESS;
15501	}
15502	return rcStrict;
15503	}
15504
15505
15506	/**
15507	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
15508	*/
15509	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15510	{
15511	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15512
15513	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15514	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
15515	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15516	AssertRCReturn(rc, rc);
15517
15518	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15519
15520	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
15521	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15522	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
15523	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15524	{
15525	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15526	rcStrict = VINF_SUCCESS;
15527	}
15528	return rcStrict;
15529	}
15530
15531
15532	/**
15533	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
15534	*/
15535	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15536	{
15537	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15538
15539	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15540	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15541	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15542	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15543	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15544	AssertRCReturn(rc, rc);
15545
15546	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15547
15548	VMXVEXITINFO ExitInfo;
15549	RT_ZERO(ExitInfo);
15550	ExitInfo.uReason = pVmxTransient->uExitReason;
15551	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15552	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15553	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15554	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15555
15556	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
15557	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15558	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15559	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15560	{
15561	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15562	rcStrict = VINF_SUCCESS;
15563	}
15564	return rcStrict;
15565	}
15566
15567	/** @} */
15568	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
15569

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 78481

Download in other formats: