HMVMXR0.cpp@ 78231

Last change on this file since 78231 was 78231, checked in by vboxsync, 6 years ago
VMM/HMVMXR0: Nested VMX: bugref:9180 Build fix.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 642.2 KB

Line
1	/* $Id: HMVMXR0.cpp 78231 2019-04-20 06:17:44Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27
28	#include <VBox/vmm/pdmapi.h>
29	#include <VBox/vmm/dbgf.h>
30	#include <VBox/vmm/iem.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#include <VBox/vmm/em.h>
35	#include <VBox/vmm/gim.h>
36	#include <VBox/vmm/apic.h>
37	#ifdef VBOX_WITH_REM
38	# include <VBox/vmm/rem.h>
39	#endif
40	#include "HMInternal.h"
41	#include <VBox/vmm/vm.h>
42	#include <VBox/vmm/hmvmxinline.h>
43	#include "HMVMXR0.h"
44	#include "dtrace/VBoxVMM.h"
45
46	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
47	#ifdef DEBUG_ramshankar
48	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
49	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
50	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
51	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
52	# define HMVMX_ALWAYS_TRAP_PF
53	# define HMVMX_ALWAYS_FLUSH_TLB
54	# define HMVMX_ALWAYS_SWAP_EFER
55	#endif
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	/** Use the function table. */
62	#define HMVMX_USE_FUNCTION_TABLE
63
64	/** Determine which tagged-TLB flush handler to use. */
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70	/** @name HMVMX_READ_XXX
71	* Flags to skip redundant reads of some common VMCS fields that are not part of
72	* the guest-CPU or VCPU state but are needed while handling VM-exits.
73	*/
74	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
75	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
76	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
77	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
78	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
79	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
80	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
81	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
82	/** @} */
83
84	/**
85	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
86	* guest using hardware-assisted VMX.
87	*
88	* This excludes state like GPRs (other than RSP) which are always are
89	* swapped and restored across the world-switch and also registers like EFER,
90	* MSR which cannot be modified by the guest without causing a VM-exit.
91	*/
92	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
93	\| CPUMCTX_EXTRN_RFLAGS \
94	\| CPUMCTX_EXTRN_RSP \
95	\| CPUMCTX_EXTRN_SREG_MASK \
96	\| CPUMCTX_EXTRN_TABLE_MASK \
97	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
98	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
99	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
100	\| CPUMCTX_EXTRN_TSC_AUX \
101	\| CPUMCTX_EXTRN_OTHER_MSRS \
102	\| CPUMCTX_EXTRN_CR0 \
103	\| CPUMCTX_EXTRN_CR3 \
104	\| CPUMCTX_EXTRN_CR4 \
105	\| CPUMCTX_EXTRN_DR7 \
106	\| CPUMCTX_EXTRN_HM_VMX_MASK)
107
108	/**
109	* Exception bitmap mask for real-mode guests (real-on-v86).
110	*
111	* We need to intercept all exceptions manually except:
112	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
113	* due to bugs in Intel CPUs.
114	* - \#PF need not be intercepted even in real-mode if we have nested paging
115	* support.
116	*/
117	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
118	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
119	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
120	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
121	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
122	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
123	\| RT_BIT(X86_XCPT_XF))
124
125	/** Maximum VM-instruction error number. */
126	#define HMVMX_INSTR_ERROR_MAX 28
127
128	/** Profiling macro. */
129	#ifdef HM_PROFILE_EXIT_DISPATCH
130	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
131	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
132	#else
133	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
134	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
135	#endif
136
137	/** Assert that preemption is disabled or covered by thread-context hooks. */
138	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
139	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
140
141	/** Assert that we haven't migrated CPUs when thread-context hooks are not
142	* used. */
143	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
144	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
145	("Illegal migration! Entered on CPU %u Current %u\n", \
146	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
147
148	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
149	* context. */
150	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
151	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
152	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
153
154	/** Helper macro for VM-exit handlers called unexpectedly. */
155	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
156	do { \
157	(a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
158	return VERR_VMX_UNEXPECTED_EXIT; \
159	} while (0)
160
161	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
162	/** Macro that does the necessary privilege checks and intercepted VM-exits for
163	* guests that attempted to execute a VMX instruction. */
164	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
165	do \
166	{ \
167	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
168	if (rcStrictTmp == VINF_SUCCESS) \
169	{ /* likely */ } \
170	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
171	{ \
172	Assert((a_pVCpu)->hm.s.Event.fPending); \
173	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
174	return VINF_SUCCESS; \
175	} \
176	else \
177	{ \
178	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
179	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
180	} \
181	} while (0)
182
183	/** Macro that decodes a memory operand for an instruction VM-exit. */
184	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
185	do \
186	{ \
187	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
188	(a_pGCPtrEffAddr)); \
189	if (rcStrictTmp == VINF_SUCCESS) \
190	{ /* likely */ } \
191	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
192	{ \
193	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
194	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
195	NOREF(uXcptTmp); \
196	return VINF_SUCCESS; \
197	} \
198	else \
199	{ \
200	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
201	return rcStrictTmp; \
202	} \
203	} while (0)
204
205	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
206
207
208	/*********************************************************************************************************************************
209	* Structures and Typedefs *
210	*********************************************************************************************************************************/
211	/**
212	* VMX transient state.
213	*
214	* A state structure for holding miscellaneous information across
215	* VMX non-root operation and restored after the transition.
216	*/
217	typedef struct VMXTRANSIENT
218	{
219	/** The host's rflags/eflags. */
220	RTCCUINTREG fEFlags;
221	#if HC_ARCH_BITS == 32
222	uint32_t u32Alignment0;
223	#endif
224	/** The guest's TPR value used for TPR shadowing. */
225	uint8_t u8GuestTpr;
226	/** Alignment. */
227	uint8_t abAlignment0[7];
228
229	/** The basic VM-exit reason. */
230	uint16_t uExitReason;
231	/** Alignment. */
232	uint16_t u16Alignment0;
233	/** The VM-exit interruption error code. */
234	uint32_t uExitIntErrorCode;
235	/** The VM-exit exit code qualification. */
236	uint64_t uExitQual;
237	/** The Guest-linear address. */
238	uint64_t uGuestLinearAddr;
239
240	/** The VM-exit interruption-information field. */
241	uint32_t uExitIntInfo;
242	/** The VM-exit instruction-length field. */
243	uint32_t cbInstr;
244	/** The VM-exit instruction-information field. */
245	VMXEXITINSTRINFO ExitInstrInfo;
246	/** Whether the VM-entry failed or not. */
247	bool fVMEntryFailed;
248	/** Whether we are currently executing a nested-guest. */
249	bool fIsNestedGuest;
250	/** Alignment. */
251	uint8_t abAlignment1[2];
252
253	/** The VM-entry interruption-information field. */
254	uint32_t uEntryIntInfo;
255	/** The VM-entry exception error code field. */
256	uint32_t uEntryXcptErrorCode;
257	/** The VM-entry instruction length field. */
258	uint32_t cbEntryInstr;
259
260	/** IDT-vectoring information field. */
261	uint32_t uIdtVectoringInfo;
262	/** IDT-vectoring error code. */
263	uint32_t uIdtVectoringErrorCode;
264
265	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
266	uint32_t fVmcsFieldsRead;
267
268	/** Whether the guest debug state was active at the time of VM-exit. */
269	bool fWasGuestDebugStateActive;
270	/** Whether the hyper debug state was active at the time of VM-exit. */
271	bool fWasHyperDebugStateActive;
272	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
273	bool fUpdatedTscOffsettingAndPreemptTimer;
274	/** Whether the VM-exit was caused by a page-fault during delivery of a
275	* contributory exception or a page-fault. */
276	bool fVectoringDoublePF;
277	/** Whether the VM-exit was caused by a page-fault during delivery of an
278	* external interrupt or NMI. */
279	bool fVectoringPF;
280	bool afAlignment0[3];
281
282	/** The VMCS info. object. */
283	PVMXVMCSINFO pVmcsInfo;
284	} VMXTRANSIENT;
285	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
286	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
287	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
288	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
289	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
290	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
291	/** Pointer to VMX transient state. */
292	typedef VMXTRANSIENT *PVMXTRANSIENT;
293
294	/**
295	* Memory operand read or write access.
296	*/
297	typedef enum VMXMEMACCESS
298	{
299	VMXMEMACCESS_READ = 0,
300	VMXMEMACCESS_WRITE = 1
301	} VMXMEMACCESS;
302
303	/**
304	* VMX VM-exit handler.
305	*
306	* @returns Strict VBox status code (i.e. informational status codes too).
307	* @param pVCpu The cross context virtual CPU structure.
308	* @param pVmxTransient The VMX-transient structure.
309	*/
310	#ifndef HMVMX_USE_FUNCTION_TABLE
311	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
312	#else
313	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
314	/** Pointer to VM-exit handler. */
315	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
316	#endif
317
318	/**
319	* VMX VM-exit handler, non-strict status code.
320	*
321	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
322	*
323	* @returns VBox status code, no informational status code returned.
324	* @param pVCpu The cross context virtual CPU structure.
325	* @param pVmxTransient The VMX-transient structure.
326	*
327	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
328	* use of that status code will be replaced with VINF_EM_SOMETHING
329	* later when switching over to IEM.
330	*/
331	#ifndef HMVMX_USE_FUNCTION_TABLE
332	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
333	#else
334	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
335	#endif
336
337
338	/*********************************************************************************************************************************
339	* Internal Functions *
340	*********************************************************************************************************************************/
341	#ifndef HMVMX_USE_FUNCTION_TABLE
342	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
343	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
344	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
345	#else
346	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
347	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
348	#endif
349	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
350	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
351	#endif
352
353	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
354	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
355	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
356	#endif
357
358	/** @name VM-exit handlers.
359	* @{
360	*/
361	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
362	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
363	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
364	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
365	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
366	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
367	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
368	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
369	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
370	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
371	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
372	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
373	static FNVMXEXITHANDLER hmR0VmxExitHlt;
374	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
375	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
376	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
377	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
378	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
379	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
380	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
381	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
382	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
383	static FNVMXEXITHANDLER hmR0VmxExitVmread;
384	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
385	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
386	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
387	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
388	#endif
389	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
390	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
391	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
392	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
393	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
394	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
395	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
396	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
397	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
398	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
399	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
400	static FNVMXEXITHANDLER hmR0VmxExitMwait;
401	static FNVMXEXITHANDLER hmR0VmxExitMtf;
402	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
403	static FNVMXEXITHANDLER hmR0VmxExitPause;
404	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
405	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
406	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
407	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
408	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
409	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
410	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
411	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
412	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
413	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
414	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
415	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
416	/** @} */
417
418	/** @name Helpers for hardware exceptions VM-exit handlers.
419	* @{
420	*/
421	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
422	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
423	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
424	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
425	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
426	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
427	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
428	/** @} */
429
430
431	/*********************************************************************************************************************************
432	* Global Variables *
433	*********************************************************************************************************************************/
434	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
435	static const uint32_t g_aVmcsCacheSegBase[] =
436	{
437	VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
438	VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
439	VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
440	VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
441	VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
442	VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
443	};
444	AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
445	#endif
446	static const uint32_t g_aVmcsSegBase[] =
447	{
448	VMX_VMCS_GUEST_ES_BASE,
449	VMX_VMCS_GUEST_CS_BASE,
450	VMX_VMCS_GUEST_SS_BASE,
451	VMX_VMCS_GUEST_DS_BASE,
452	VMX_VMCS_GUEST_FS_BASE,
453	VMX_VMCS_GUEST_GS_BASE
454	};
455	static const uint32_t g_aVmcsSegSel[] =
456	{
457	VMX_VMCS16_GUEST_ES_SEL,
458	VMX_VMCS16_GUEST_CS_SEL,
459	VMX_VMCS16_GUEST_SS_SEL,
460	VMX_VMCS16_GUEST_DS_SEL,
461	VMX_VMCS16_GUEST_FS_SEL,
462	VMX_VMCS16_GUEST_GS_SEL
463	};
464	static const uint32_t g_aVmcsSegLimit[] =
465	{
466	VMX_VMCS32_GUEST_ES_LIMIT,
467	VMX_VMCS32_GUEST_CS_LIMIT,
468	VMX_VMCS32_GUEST_SS_LIMIT,
469	VMX_VMCS32_GUEST_DS_LIMIT,
470	VMX_VMCS32_GUEST_FS_LIMIT,
471	VMX_VMCS32_GUEST_GS_LIMIT
472	};
473	static const uint32_t g_aVmcsSegAttr[] =
474	{
475	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
476	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
477	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
478	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
479	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
480	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
481	};
482	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
483	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
484	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
485	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
486
487	#ifdef HMVMX_USE_FUNCTION_TABLE
488	/**
489	* VMX_EXIT dispatch table.
490	*/
491	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
492	{
493	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
494	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
495	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
496	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
497	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
498	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
499	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
500	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
501	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
502	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
503	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
504	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
505	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
506	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
507	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
508	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
509	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
510	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
511	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
512	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
513	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
514	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
515	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
516	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
517	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
518	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
519	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
520	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
521	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
522	#else
523	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
524	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
525	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
526	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
527	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
528	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
529	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
530	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
531	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
532	#endif
533	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
534	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
535	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
536	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
537	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
538	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
539	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
540	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
541	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
542	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
543	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
544	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
545	/* 40 UNDEFINED */ hmR0VmxExitPause,
546	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
547	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
548	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
549	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
550	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
551	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitXdtrAccess,
552	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitXdtrAccess,
553	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
554	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
555	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
556	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
557	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
558	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
559	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
560	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
561	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
562	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
563	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
564	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
565	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
566	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
567	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
568	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
569	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
570	};
571	#endif /* HMVMX_USE_FUNCTION_TABLE */
572
573	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
574	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
575	{
576	/* 0 */ "(Not Used)",
577	/* 1 */ "VMCALL executed in VMX root operation.",
578	/* 2 */ "VMCLEAR with invalid physical address.",
579	/* 3 */ "VMCLEAR with VMXON pointer.",
580	/* 4 */ "VMLAUNCH with non-clear VMCS.",
581	/* 5 */ "VMRESUME with non-launched VMCS.",
582	/* 6 */ "VMRESUME after VMXOFF",
583	/* 7 */ "VM-entry with invalid control fields.",
584	/* 8 */ "VM-entry with invalid host state fields.",
585	/* 9 */ "VMPTRLD with invalid physical address.",
586	/* 10 */ "VMPTRLD with VMXON pointer.",
587	/* 11 */ "VMPTRLD with incorrect revision identifier.",
588	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
589	/* 13 */ "VMWRITE to read-only VMCS component.",
590	/* 14 */ "(Not Used)",
591	/* 15 */ "VMXON executed in VMX root operation.",
592	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
593	/* 17 */ "VM-entry with non-launched executing VMCS.",
594	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
595	/* 19 */ "VMCALL with non-clear VMCS.",
596	/* 20 */ "VMCALL with invalid VM-exit control fields.",
597	/* 21 */ "(Not Used)",
598	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
599	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
600	/* 24 */ "VMCALL with invalid SMM-monitor features.",
601	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
602	/* 26 */ "VM-entry with events blocked by MOV SS.",
603	/* 27 */ "(Not Used)",
604	/* 28 */ "Invalid operand to INVEPT/INVVPID."
605	};
606	#endif /* VBOX_STRICT */
607
608
609	/**
610	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
611	*
612	* Any bit set in this mask is owned by the host/hypervisor and would cause a
613	* VM-exit when modified by the guest.
614	*
615	* @returns The static CR0 guest/host mask.
616	* @param pVCpu The cross context virtual CPU structure.
617	*/
618	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PVMCPU pVCpu)
619	{
620	/*
621	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
622	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
623	*/
624	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
625	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
626	* and @bugref{6944}. */
627	PVM pVM = pVCpu->CTX_SUFF(pVM);
628	return ( X86_CR0_PE
629	\| X86_CR0_NE
630	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
631	\| X86_CR0_PG
632	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
633	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
634	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
635	}
636
637
638	/**
639	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
640	*
641	* Any bit set in this mask is owned by the host/hypervisor and would cause a
642	* VM-exit when modified by the guest.
643	*
644	* @returns The static CR4 guest/host mask.
645	* @param pVCpu The cross context virtual CPU structure.
646	*/
647	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PVMCPU pVCpu)
648	{
649	/*
650	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
651	* these bits are reserved on hardware that does not support them. Since the
652	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
653	* these bits and handle it depending on whether we expose them to the guest.
654	*/
655	PVM pVM = pVCpu->CTX_SUFF(pVM);
656	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
657	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
658	return ( X86_CR4_VMXE
659	\| X86_CR4_VME
660	\| X86_CR4_PAE
661	\| X86_CR4_PGE
662	\| X86_CR4_PSE
663	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
664	\| (fPcid ? X86_CR4_PCIDE : 0));
665	}
666
667
668	/**
669	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
670	* area.
671	*
672	* @returns @c true if it's different, @c false otherwise.
673	* @param pVmcsInfo The VMCS info. object.
674	*/
675	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
676	{
677	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
678	&& pVmcsInfo->pvGuestMsrStore);
679	}
680
681
682	/**
683	* Adds one or more exceptions to the exception bitmap and commits it to the current
684	* VMCS.
685	*
686	* @returns VBox status code.
687	* @param pVmxTransient The VMX-transient structure.
688	* @param uXcptMask The exception(s) to add.
689	*/
690	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
691	{
692	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
693	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
694	if ((uXcptBitmap & uXcptMask) != uXcptMask)
695	{
696	uXcptBitmap \|= uXcptMask;
697	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
698	AssertRCReturn(rc, rc);
699	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
700	}
701	return VINF_SUCCESS;
702	}
703
704
705	/**
706	* Adds an exception to the exception bitmap and commits it to the current VMCS.
707	*
708	* @returns VBox status code.
709	* @param pVmxTransient The VMX-transient structure.
710	* @param uXcpt The exception to add.
711	*/
712	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
713	{
714	Assert(uXcpt <= X86_XCPT_LAST);
715	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
716	}
717
718
719	/**
720	* Remove one or more exceptions from the exception bitmap and commits it to the
721	* current VMCS.
722	*
723	* This takes care of not removing the exception intercept if a nested-guest
724	* requires the exception to be intercepted.
725	*
726	* @returns VBox status code.
727	* @param pVCpu The cross context virtual CPU structure.
728	* @param pVmxTransient The VMX-transient structure.
729	* @param uXcptMask The exception(s) to remove.
730	*/
731	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
732	{
733	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
734	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
735	if (u32XcptBitmap & uXcptMask)
736	{
737	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
738	if (!pVmxTransient->fIsNestedGuest)
739	{ /* likely */ }
740	else
741	{
742	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
743	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
744	}
745	#endif
746	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
747	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
748	\| RT_BIT(X86_XCPT_DE)
749	\| RT_BIT(X86_XCPT_NM)
750	\| RT_BIT(X86_XCPT_TS)
751	\| RT_BIT(X86_XCPT_UD)
752	\| RT_BIT(X86_XCPT_NP)
753	\| RT_BIT(X86_XCPT_SS)
754	\| RT_BIT(X86_XCPT_GP)
755	\| RT_BIT(X86_XCPT_PF)
756	\| RT_BIT(X86_XCPT_MF));
757	#elif defined(HMVMX_ALWAYS_TRAP_PF)
758	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
759	#endif
760	if (uXcptMask)
761	{
762	/* Validate we are not removing any essential exception intercepts. */
763	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
764	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
765	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
766
767	/* Remove it from the exception bitmap. */
768	u32XcptBitmap &= ~uXcptMask;
769
770	/* Commit and update the cache if necessary. */
771	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
772	{
773	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
774	AssertRCReturn(rc, rc);
775	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
776	}
777	}
778	}
779	return VINF_SUCCESS;
780	}
781
782
783	/**
784	* Remove an exceptions from the exception bitmap and commits it to the current
785	* VMCS.
786	*
787	* @returns VBox status code.
788	* @param pVCpu The cross context virtual CPU structure.
789	* @param pVmxTransient The VMX-transient structure.
790	* @param uXcpt The exception to remove.
791	*/
792	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
793	{
794	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
795	}
796
797
798	/**
799	* Loads the VMCS specified by the VMCS info. object.
800	*
801	* @returns VBox status code.
802	* @param pVmcsInfo The VMCS info. object.
803	*/
804	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
805	{
806	Assert(pVmcsInfo);
807	Assert(pVmcsInfo->HCPhysVmcs);
808	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
809
810	if (pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_CLEAR)
811	{
812	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
813	if (RT_SUCCESS(rc))
814	{
815	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
816	return VINF_SUCCESS;
817	}
818	return rc;
819	}
820	return VERR_VMX_INVALID_VMCS_LAUNCH_STATE;
821	}
822
823
824	/**
825	* Clears the VMCS specified by the VMCS info. object.
826	*
827	* @returns VBox status code.
828	* @param pVmcsInfo The VMCS info. object.
829	*/
830	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
831	{
832	Assert(pVmcsInfo);
833	Assert(pVmcsInfo->HCPhysVmcs);
834	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
835
836	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
837	if (RT_SUCCESS(rc))
838	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
839	return rc;
840	}
841
842
843	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
844	/**
845	* Switches the current VMCS to the one specified.
846	*
847	* @returns VBox status code.
848	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
849	* @param pVmcsInfoTo The VMCS info. object we are switching to.
850	*
851	* @remarks Called with interrupts disabled.
852	*/
853	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
854	{
855	Assert(pVmcsInfoFrom);
856	Assert(pVmcsInfoTo);
857
858	/*
859	* Clear the VMCS we are switching out if it has not already been cleared.
860	* This will sync any CPU internal data back to the VMCS.
861	*/
862	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
863	{
864	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
865	if (RT_SUCCESS(rc))
866	{ /* likely */ }
867	else
868	return rc;
869	}
870
871	/*
872	* Clear the VMCS we are switching to if it has not already been cleared.
873	* This will initialize the VMCS launch state to "clear" required for loading it.
874	*
875	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
876	*/
877	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
878	{
879	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
880	if (RT_SUCCESS(rc))
881	{ /* likely */ }
882	else
883	return rc;
884	}
885
886	/*
887	* Finally, load the VMCS we are switching to.
888	*/
889	return hmR0VmxLoadVmcs(pVmcsInfoTo);
890	}
891	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
892
893
894	/**
895	* Updates the VM's last error record.
896	*
897	* If there was a VMX instruction error, reads the error data from the VMCS and
898	* updates VCPU's last error record as well.
899	*
900	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
901	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
902	* VERR_VMX_INVALID_VMCS_FIELD.
903	* @param rc The error code.
904	*/
905	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
906	{
907	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
908	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
909	{
910	AssertPtrReturnVoid(pVCpu);
911	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
912	}
913	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
914	}
915
916
917	#ifdef VBOX_STRICT
918	/**
919	* Reads the VM-entry interruption-information field from the VMCS into the VMX
920	* transient structure.
921	*
922	* @returns VBox status code.
923	* @param pVmxTransient The VMX-transient structure.
924	*
925	* @remarks No-long-jump zone!!!
926	*/
927	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
928	{
929	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
930	AssertRCReturn(rc, rc);
931	return VINF_SUCCESS;
932	}
933
934
935	/**
936	* Reads the VM-entry exception error code field from the VMCS into
937	* the VMX transient structure.
938	*
939	* @returns VBox status code.
940	* @param pVmxTransient The VMX-transient structure.
941	*
942	* @remarks No-long-jump zone!!!
943	*/
944	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
945	{
946	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
947	AssertRCReturn(rc, rc);
948	return VINF_SUCCESS;
949	}
950
951
952	/**
953	* Reads the VM-entry exception error code field from the VMCS into
954	* the VMX transient structure.
955	*
956	* @returns VBox status code.
957	* @param pVmxTransient The VMX-transient structure.
958	*
959	* @remarks No-long-jump zone!!!
960	*/
961	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
962	{
963	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
964	AssertRCReturn(rc, rc);
965	return VINF_SUCCESS;
966	}
967	#endif /* VBOX_STRICT */
968
969
970	/**
971	* Reads the VM-exit interruption-information field from the VMCS into the VMX
972	* transient structure.
973	*
974	* @returns VBox status code.
975	* @param pVmxTransient The VMX-transient structure.
976	*/
977	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
978	{
979	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
980	{
981	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
982	AssertRCReturn(rc,rc);
983	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
984	}
985	return VINF_SUCCESS;
986	}
987
988
989	/**
990	* Reads the VM-exit interruption error code from the VMCS into the VMX
991	* transient structure.
992	*
993	* @returns VBox status code.
994	* @param pVmxTransient The VMX-transient structure.
995	*/
996	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
997	{
998	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
999	{
1000	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1001	AssertRCReturn(rc, rc);
1002	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1003	}
1004	return VINF_SUCCESS;
1005	}
1006
1007
1008	/**
1009	* Reads the VM-exit instruction length field from the VMCS into the VMX
1010	* transient structure.
1011	*
1012	* @returns VBox status code.
1013	* @param pVmxTransient The VMX-transient structure.
1014	*/
1015	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1016	{
1017	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1018	{
1019	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1020	AssertRCReturn(rc, rc);
1021	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1022	}
1023	return VINF_SUCCESS;
1024	}
1025
1026
1027	/**
1028	* Reads the VM-exit instruction-information field from the VMCS into
1029	* the VMX transient structure.
1030	*
1031	* @returns VBox status code.
1032	* @param pVmxTransient The VMX-transient structure.
1033	*/
1034	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1035	{
1036	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1037	{
1038	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1039	AssertRCReturn(rc, rc);
1040	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1041	}
1042	return VINF_SUCCESS;
1043	}
1044
1045
1046	/**
1047	* Reads the VM-exit Qualification from the VMCS into the VMX transient structure.
1048	*
1049	* @returns VBox status code.
1050	* @param pVCpu The cross context virtual CPU structure of the
1051	* calling EMT. (Required for the VMCS cache case.)
1052	* @param pVmxTransient The VMX-transient structure.
1053	*/
1054	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1055	{
1056	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1057	{
1058	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1059	AssertRCReturn(rc, rc);
1060	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1061	}
1062	return VINF_SUCCESS;
1063	}
1064
1065
1066	/**
1067	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1068	*
1069	* @returns VBox status code.
1070	* @param pVCpu The cross context virtual CPU structure of the
1071	* calling EMT. (Required for the VMCS cache case.)
1072	* @param pVmxTransient The VMX-transient structure.
1073	*/
1074	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1075	{
1076	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1077	{
1078	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1079	AssertRCReturn(rc, rc);
1080	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1081	}
1082	return VINF_SUCCESS;
1083	}
1084
1085
1086	/**
1087	* Reads the IDT-vectoring information field from the VMCS into the VMX
1088	* transient structure.
1089	*
1090	* @returns VBox status code.
1091	* @param pVmxTransient The VMX-transient structure.
1092	*
1093	* @remarks No-long-jump zone!!!
1094	*/
1095	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1096	{
1097	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1098	{
1099	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1100	AssertRCReturn(rc, rc);
1101	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1102	}
1103	return VINF_SUCCESS;
1104	}
1105
1106
1107	/**
1108	* Reads the IDT-vectoring error code from the VMCS into the VMX
1109	* transient structure.
1110	*
1111	* @returns VBox status code.
1112	* @param pVmxTransient The VMX-transient structure.
1113	*/
1114	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1115	{
1116	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1117	{
1118	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1119	AssertRCReturn(rc, rc);
1120	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1121	}
1122	return VINF_SUCCESS;
1123	}
1124
1125
1126	/**
1127	* Enters VMX root mode operation on the current CPU.
1128	*
1129	* @returns VBox status code.
1130	* @param pVM The cross context VM structure. Can be
1131	* NULL, after a resume.
1132	* @param HCPhysCpuPage Physical address of the VMXON region.
1133	* @param pvCpuPage Pointer to the VMXON region.
1134	*/
1135	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1136	{
1137	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1138	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1139	Assert(pvCpuPage);
1140	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1141
1142	if (pVM)
1143	{
1144	/* Write the VMCS revision identifier to the VMXON region. */
1145	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1146	}
1147
1148	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1149	RTCCUINTREG fEFlags = ASMIntDisableFlags();
1150
1151	/* Enable the VMX bit in CR4 if necessary. */
1152	RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1153
1154	/* Enter VMX root mode. */
1155	int rc = VMXEnable(HCPhysCpuPage);
1156	if (RT_FAILURE(rc))
1157	{
1158	if (!(uOldCr4 & X86_CR4_VMXE))
1159	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
1160
1161	if (pVM)
1162	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1163	}
1164
1165	/* Restore interrupts. */
1166	ASMSetFlags(fEFlags);
1167	return rc;
1168	}
1169
1170
1171	/**
1172	* Exits VMX root mode operation on the current CPU.
1173	*
1174	* @returns VBox status code.
1175	*/
1176	static int hmR0VmxLeaveRootMode(void)
1177	{
1178	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1179
1180	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1181	RTCCUINTREG fEFlags = ASMIntDisableFlags();
1182
1183	/* If we're for some reason not in VMX root mode, then don't leave it. */
1184	RTCCUINTREG uHostCR4 = ASMGetCR4();
1185
1186	int rc;
1187	if (uHostCR4 & X86_CR4_VMXE)
1188	{
1189	/* Exit VMX root mode and clear the VMX bit in CR4. */
1190	VMXDisable();
1191	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
1192	rc = VINF_SUCCESS;
1193	}
1194	else
1195	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1196
1197	/* Restore interrupts. */
1198	ASMSetFlags(fEFlags);
1199	return rc;
1200	}
1201
1202
1203	/**
1204	* Allocates and maps a physically contiguous page. The allocated page is
1205	* zero'd out (used by various VT-x structures).
1206	*
1207	* @returns IPRT status code.
1208	* @param pMemObj Pointer to the ring-0 memory object.
1209	* @param ppVirt Where to store the virtual address of the
1210	* allocation.
1211	* @param pHCPhys Where to store the physical address of the
1212	* allocation.
1213	*/
1214	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1215	{
1216	AssertPtr(pMemObj);
1217	AssertPtr(ppVirt);
1218	AssertPtr(pHCPhys);
1219	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1220	if (RT_FAILURE(rc))
1221	return rc;
1222	ppVirt = RTR0MemObjAddress(pMemObj);
1223	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1224	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1225	return VINF_SUCCESS;
1226	}
1227
1228
1229	/**
1230	* Frees and unmaps an allocated, physical page.
1231	*
1232	* @param pMemObj Pointer to the ring-0 memory object.
1233	* @param ppVirt Where to re-initialize the virtual address of
1234	* allocation as 0.
1235	* @param pHCPhys Where to re-initialize the physical address of the
1236	* allocation as 0.
1237	*/
1238	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1239	{
1240	AssertPtr(pMemObj);
1241	AssertPtr(ppVirt);
1242	AssertPtr(pHCPhys);
1243	/* NULL is valid, accepted and ignored by the free function below. */
1244	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1245	*pMemObj = NIL_RTR0MEMOBJ;
1246	*ppVirt = NULL;
1247	*pHCPhys = NIL_RTHCPHYS;
1248	}
1249
1250
1251	/**
1252	* Initializes a VMCS info. object.
1253	*
1254	* @param pVmcsInfo The VMCS info. object.
1255	*/
1256	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1257	{
1258	RT_ZERO(*pVmcsInfo);
1259
1260	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1261	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1262	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1263	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1264	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1265	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1266	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1267	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1268	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1269	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1270	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1271	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1272	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1273	}
1274
1275
1276	/**
1277	* Frees the VT-x structures for a VMCS info. object.
1278	*
1279	* @param pVM The cross context VM structure.
1280	* @param pVmcsInfo The VMCS info. object.
1281	*/
1282	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1283	{
1284	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1285
1286	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1287	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1288
1289	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1290	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1291	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1292
1293	hmR0VmxInitVmcsInfo(pVmcsInfo);
1294	}
1295
1296
1297	/**
1298	* Allocates the VT-x structures for a VMCS info. object.
1299	*
1300	* @returns VBox status code.
1301	* @param pVCpu The cross context virtual CPU structure.
1302	* @param pVmcsInfo The VMCS info. object.
1303	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1304	*/
1305	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1306	{
1307	PVM pVM = pVCpu->CTX_SUFF(pVM);
1308
1309	/* Allocate the guest VM control structure (VMCS). */
1310	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1311	if (RT_SUCCESS(rc))
1312	{
1313	if (!fIsNstGstVmcs)
1314	{
1315	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1316	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1317	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1318	{
1319	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1320	NULL /* pR3Ptr /, NULL / pRCPtr */);
1321	}
1322	}
1323	else
1324	{
1325	Assert(pVmcsInfo->HCPhysVirtApic == NIL_RTHCPHYS);
1326	Assert(!pVmcsInfo->pbVirtApic);
1327	}
1328
1329	if (RT_SUCCESS(rc))
1330	{
1331	/*
1332	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1333	* transparent accesses of specific MSRs.
1334	*
1335	* If the condition for enabling MSR bitmaps changes here, don't forget to
1336	* update HMIsMsrBitmapActive().
1337	*
1338	* We don't share MSR bitmaps between the guest and nested-guest as we then
1339	* don't need to care about carefully restoring the guest MSR bitmap.
1340	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1341	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1342	*/
1343	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1344	{
1345	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1346	if (RT_SUCCESS(rc))
1347	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1348	}
1349
1350	if (RT_SUCCESS(rc))
1351	{
1352	/*
1353	* Allocate the VM-entry MSR-load area for the guest MSRs.
1354	*
1355	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1356	* the guest and nested-guest.
1357	*/
1358	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1359	&pVmcsInfo->HCPhysGuestMsrLoad);
1360	if (RT_SUCCESS(rc))
1361	{
1362	/*
1363	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1364	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1365	*/
1366	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1367	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1368	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1369
1370	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1371	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1372	&pVmcsInfo->HCPhysHostMsrLoad);
1373	}
1374	}
1375	}
1376	}
1377
1378	return rc;
1379	}
1380
1381
1382	/**
1383	* Free all VT-x structures for the VM.
1384	*
1385	* @returns IPRT status code.
1386	* @param pVM The cross context VM structure.
1387	*/
1388	static void hmR0VmxStructsFree(PVM pVM)
1389	{
1390	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1391	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1392	#endif
1393	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1394
1395	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1396	{
1397	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1398	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1399	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1400	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1401	if (pVM->cpum.ro.GuestFeatures.fVmx)
1402	{
1403	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1404	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1405	}
1406	#endif
1407	}
1408	}
1409
1410
1411	/**
1412	* Allocate all VT-x structures for the VM.
1413	*
1414	* @returns IPRT status code.
1415	* @param pVM The cross context VM structure.
1416	*/
1417	static int hmR0VmxStructsAlloc(PVM pVM)
1418	{
1419	/*
1420	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1421	* The VMCS size cannot be more than 4096 bytes.
1422	*
1423	* See Intel spec. Appendix A.1 "Basic VMX Information".
1424	*/
1425	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1426	if (cbVmcs <= X86_PAGE_4K_SIZE)
1427	{ /* likely */ }
1428	else
1429	{
1430	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1431	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1432	}
1433
1434	/*
1435	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1436	*/
1437	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1438	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1439	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1440	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1441	#endif
1442
1443	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1444	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1445	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1446
1447	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1448	{
1449	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1450	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1451	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1452	}
1453
1454	/*
1455	* Allocate per-VM VT-x structures.
1456	*/
1457	int rc = VINF_SUCCESS;
1458	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1459	/* Allocate crash-dump magic scratch page. */
1460	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1461	if (RT_FAILURE(rc))
1462	{
1463	hmR0VmxStructsFree(pVM);
1464	return rc;
1465	}
1466	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1467	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1468	#endif
1469
1470	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1471	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1472	{
1473	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1474	&pVM->hm.s.vmx.HCPhysApicAccess);
1475	if (RT_FAILURE(rc))
1476	{
1477	hmR0VmxStructsFree(pVM);
1478	return rc;
1479	}
1480	}
1481
1482	/*
1483	* Initialize per-VCPU VT-x structures.
1484	*/
1485	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1486	{
1487	/* Allocate the guest VMCS structures. */
1488	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1489	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
1490	if (RT_SUCCESS(rc))
1491	{
1492	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1493	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
1494	if (pVM->cpum.ro.GuestFeatures.fVmx)
1495	{
1496	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
1497	if (RT_SUCCESS(rc))
1498	{ /* likely */ }
1499	else
1500	break;
1501	}
1502	#endif
1503	}
1504	else
1505	break;
1506	}
1507
1508	if (RT_FAILURE(rc))
1509	{
1510	hmR0VmxStructsFree(pVM);
1511	return rc;
1512	}
1513
1514	return VINF_SUCCESS;
1515	}
1516
1517
1518	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1519	/**
1520	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
1521	*
1522	* @returns @c true if the MSR is intercepted, @c false otherwise.
1523	* @param pvMsrBitmap The MSR bitmap.
1524	* @param offMsr The MSR byte offset.
1525	* @param iBit The bit offset from the byte offset.
1526	*/
1527	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
1528	{
1529	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
1530	Assert(pbMsrBitmap);
1531	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
1532	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
1533	}
1534	#endif
1535
1536
1537	/**
1538	* Sets the permission bits for the specified MSR in the given MSR bitmap.
1539	*
1540	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
1541	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
1542	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
1543	* the read/write access of this MSR.
1544	*
1545	* @param pVCpu The cross context virtual CPU structure.
1546	* @param pVmcsInfo The VMCS info. object.
1547	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1548	* @param idMsr The MSR value.
1549	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
1550	* include both a read -and- a write permission!
1551	*
1552	* @sa HMGetVmxMsrPermission.
1553	*/
1554	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
1555	{
1556	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
1557	Assert(pbMsrBitmap);
1558	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
1559
1560	/*
1561	* MSR-bitmap Layout:
1562	* Byte index MSR range Interpreted as
1563	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1564	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1565	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1566	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1567	*
1568	* A bit corresponding to an MSR within the above range causes a VM-exit
1569	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
1570	* the MSR range, it always cause a VM-exit.
1571	*
1572	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
1573	*/
1574	uint16_t const offBitmapRead = 0;
1575	uint16_t const offBitmapWrite = 0x800;
1576	uint16_t offMsr;
1577	int32_t iBit;
1578	if (idMsr <= UINT32_C(0x00001fff))
1579	{
1580	offMsr = 0;
1581	iBit = idMsr;
1582	}
1583	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1584	{
1585	offMsr = 0x400;
1586	iBit = idMsr - UINT32_C(0xc0000000);
1587	}
1588	else
1589	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
1590
1591	/*
1592	* Set the MSR read permission.
1593	*/
1594	uint16_t const offMsrRead = offBitmapRead + offMsr;
1595	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
1596	if (fMsrpm & VMXMSRPM_ALLOW_RD)
1597	{
1598	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1599	bool const fClear = !fIsNstGstVmcs ? true
1600	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
1601	#else
1602	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1603	bool const fClear = true;
1604	#endif
1605	if (fClear)
1606	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
1607	}
1608	else
1609	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
1610
1611	/*
1612	* Set the MSR write permission.
1613	*/
1614	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
1615	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
1616	if (fMsrpm & VMXMSRPM_ALLOW_WR)
1617	{
1618	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1619	bool const fClear = !fIsNstGstVmcs ? true
1620	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
1621	#else
1622	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1623	bool const fClear = true;
1624	#endif
1625	if (fClear)
1626	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
1627	}
1628	else
1629	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
1630	}
1631
1632
1633	/**
1634	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1635	* area.
1636	*
1637	* @returns VBox status code.
1638	* @param pVCpu The cross context virtual CPU structure.
1639	* @param pVmcsInfo The VMCS info. object.
1640	* @param cMsrs The number of MSRs.
1641	*/
1642	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
1643	{
1644	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1645	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1646	if (RT_UNLIKELY(cMsrs >= cMaxSupportedMsrs))
1647	{
1648	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1649	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1650	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1651	}
1652
1653	/* Commit the MSR counts to the VMCS and update the cache. */
1654	int rc = VINF_SUCCESS;
1655	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
1656	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1657	if (pVmcsInfo->cExitMsrStore != cMsrs)
1658	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1659	if (pVmcsInfo->cExitMsrLoad != cMsrs)
1660	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1661	AssertRCReturn(rc, rc);
1662
1663	pVmcsInfo->cEntryMsrLoad = cMsrs;
1664	pVmcsInfo->cExitMsrStore = cMsrs;
1665	pVmcsInfo->cExitMsrLoad = cMsrs;
1666
1667	return VINF_SUCCESS;
1668	}
1669
1670
1671	/**
1672	* Adds a new (or updates the value of an existing) guest/host MSR
1673	* pair to be swapped during the world-switch as part of the
1674	* auto-load/store MSR area in the VMCS.
1675	*
1676	* @returns VBox status code.
1677	* @param pVCpu The cross context virtual CPU structure.
1678	* @param pVmxTransient The VMX-transient structure.
1679	* @param idMsr The MSR.
1680	* @param uGuestMsrValue Value of the guest MSR.
1681	* @param fSetReadWrite Whether to set the guest read/write access of this
1682	* MSR (thus not causing a VM-exit).
1683	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1684	* necessary.
1685	*/
1686	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
1687	bool fSetReadWrite, bool fUpdateHostMsr)
1688	{
1689	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1690	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1691	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1692	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1693	uint32_t i;
1694
1695	/* Paranoia. */
1696	Assert(pGuestMsrLoad);
1697
1698	/* Check if the MSR already exists in the VM-entry MSR-load area. */
1699	for (i = 0; i < cMsrs; i++)
1700	{
1701	if (pGuestMsrLoad->u32Msr == idMsr)
1702	break;
1703	pGuestMsrLoad++;
1704	}
1705
1706	bool fAdded = false;
1707	if (i == cMsrs)
1708	{
1709	/* The MSR does not exist, bump the MSR coun to make room for the new MSR. */
1710	++cMsrs;
1711	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1712	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
1713
1714	/* Set the guest to read/write this MSR without causing VM-exits. */
1715	if ( fSetReadWrite
1716	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
1717	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
1718
1719	fAdded = true;
1720	}
1721
1722	/* Update the MSR value for the newly added or already existing MSR. */
1723	pGuestMsrLoad->u32Msr = idMsr;
1724	pGuestMsrLoad->u64Value = uGuestMsrValue;
1725
1726	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
1727	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1728	{
1729	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1730	pGuestMsrStore += i;
1731	pGuestMsrStore->u32Msr = idMsr;
1732	pGuestMsrStore->u64Value = 0;
1733	}
1734
1735	/* Update the corresponding slot in the host MSR area. */
1736	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1737	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad && pHostMsr != pVmcsInfo->pvGuestMsrStore);
1738	pHostMsr += i;
1739	pHostMsr->u32Msr = idMsr;
1740
1741	/*
1742	* Only if the caller requests to update the host MSR value AND we've newly added the
1743	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
1744	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
1745	*
1746	* We do this for performance reasons since reading MSRs may be quite expensive.
1747	*/
1748	if ( fAdded
1749	&& fUpdateHostMsr)
1750	{
1751	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1752	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1753	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1754	}
1755	return VINF_SUCCESS;
1756	}
1757
1758
1759	/**
1760	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1761	* auto-load/store MSR area in the VMCS.
1762	*
1763	* @returns VBox status code.
1764	* @param pVCpu The cross context virtual CPU structure.
1765	* @param pVmxTransient The VMX-transient structure.
1766	* @param idMsr The MSR.
1767	*/
1768	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
1769	{
1770	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1771	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1772	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1773	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1774
1775	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
1776	for (uint32_t i = 0; i < cMsrs; i++)
1777	{
1778	/* Find the MSR. */
1779	if (pGuestMsrLoad->u32Msr == idMsr)
1780	{
1781	/* If it's the last MSR, simply reduce the count. */
1782	if (i == cMsrs - 1)
1783	{
1784	--cMsrs;
1785	break;
1786	}
1787
1788	/* Remove it by copying the last MSR in place of it, and reducing the count. */
1789	PVMXAUTOMSR pLastGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1790	pLastGuestMsrLoad += cMsrs - 1;
1791	pGuestMsrLoad->u32Msr = pLastGuestMsrLoad->u32Msr;
1792	pGuestMsrLoad->u64Value = pLastGuestMsrLoad->u64Value;
1793
1794	/* Remove it from the VM-exit MSR-store area if we are using a different page. */
1795	if (fSeparateExitMsrStorePage)
1796	{
1797	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1798	PVMXAUTOMSR pLastGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1799	pGuestMsrStore += i;
1800	pLastGuestMsrStore += cMsrs - 1;
1801	Assert(pGuestMsrStore->u32Msr == idMsr);
1802	pGuestMsrStore->u32Msr = pLastGuestMsrStore->u32Msr;
1803	pGuestMsrStore->u64Value = pLastGuestMsrStore->u64Value;
1804	}
1805
1806	/* Remove it from the VM-exit MSR-load area. */
1807	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1808	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1809	pHostMsr += i;
1810	pLastHostMsr += cMsrs - 1;
1811	Assert(pHostMsr->u32Msr == idMsr);
1812	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1813	pHostMsr->u64Value = pLastHostMsr->u64Value;
1814	--cMsrs;
1815	break;
1816	}
1817	pGuestMsrLoad++;
1818	}
1819
1820	/* Update the VMCS if the count changed (meaning the MSR was found). */
1821	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
1822	{
1823	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1824	AssertRCReturn(rc, rc);
1825
1826	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1827	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1828	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
1829
1830	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
1831	return VINF_SUCCESS;
1832	}
1833
1834	return VERR_NOT_FOUND;
1835	}
1836
1837
1838	/**
1839	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
1840	*
1841	* @returns @c true if found, @c false otherwise.
1842	* @param pVmcsInfo The VMCS info. object.
1843	* @param idMsr The MSR to find.
1844	*/
1845	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
1846	{
1847	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1848	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
1849	for (uint32_t i = 0; i < cMsrs; i++)
1850	{
1851	if (pGuestMsrLoad->u32Msr == idMsr)
1852	return true;
1853	pGuestMsrLoad++;
1854	}
1855	return false;
1856	}
1857
1858
1859	/**
1860	* Updates the value of all host MSRs in the VM-exit MSR-load area.
1861	*
1862	* @param pVCpu The cross context virtual CPU structure.
1863	* @param pVmcsInfo The VMCS info. object.
1864	*
1865	* @remarks No-long-jump zone!!!
1866	*/
1867	static void hmR0VmxUpdateAutoLoadHostMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
1868	{
1869	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1870	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
1871
1872	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1873	Assert(pHostMsrLoad);
1874
1875	for (uint32_t i = 0; i < cMsrs; i++, pHostMsrLoad++)
1876	{
1877	/*
1878	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1879	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1880	*/
1881	if (pHostMsrLoad->u32Msr == MSR_K6_EFER)
1882	pHostMsrLoad->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
1883	else
1884	pHostMsrLoad->u64Value = ASMRdMsr(pHostMsrLoad->u32Msr);
1885	}
1886	}
1887
1888
1889	/**
1890	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1891	* perform lazy restoration of the host MSRs while leaving VT-x.
1892	*
1893	* @param pVCpu The cross context virtual CPU structure.
1894	*
1895	* @remarks No-long-jump zone!!!
1896	*/
1897	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1898	{
1899	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1900
1901	/*
1902	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
1903	*/
1904	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1905	{
1906	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1907	#if HC_ARCH_BITS == 64
1908	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1909	{
1910	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
1911	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
1912	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
1913	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1914	}
1915	#endif
1916	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1917	}
1918	}
1919
1920
1921	/**
1922	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1923	* lazily while leaving VT-x.
1924	*
1925	* @returns true if it does, false otherwise.
1926	* @param pVCpu The cross context virtual CPU structure.
1927	* @param idMsr The MSR to check.
1928	*/
1929	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t idMsr)
1930	{
1931	NOREF(pVCpu);
1932	#if HC_ARCH_BITS == 64
1933	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1934	{
1935	switch (idMsr)
1936	{
1937	case MSR_K8_LSTAR:
1938	case MSR_K6_STAR:
1939	case MSR_K8_SF_MASK:
1940	case MSR_K8_KERNEL_GS_BASE:
1941	return true;
1942	}
1943	}
1944	#else
1945	RT_NOREF(pVCpu, idMsr);
1946	#endif
1947	return false;
1948	}
1949
1950
1951	/**
1952	* Loads a set of guests MSRs to allow read/passthru to the guest.
1953	*
1954	* The name of this function is slightly confusing. This function does NOT
1955	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1956	* common prefix for functions dealing with "lazy restoration" of the shared
1957	* MSRs.
1958	*
1959	* @param pVCpu The cross context virtual CPU structure.
1960	*
1961	* @remarks No-long-jump zone!!!
1962	*/
1963	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1964	{
1965	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1966	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1967
1968	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1969	#if HC_ARCH_BITS == 64
1970	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1971	{
1972	/*
1973	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1974	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1975	* we can skip a few MSR writes.
1976	*
1977	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1978	* guest MSR values in the guest-CPU context might be different to what's currently
1979	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1980	* CPU, see @bugref{8728}.
1981	*/
1982	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1983	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1984	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
1985	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
1986	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
1987	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
1988	{
1989	#ifdef VBOX_STRICT
1990	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1991	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1992	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1993	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1994	#endif
1995	}
1996	else
1997	{
1998	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1999	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2000	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2001	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2002	}
2003	}
2004	#endif
2005	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2006	}
2007
2008
2009	/**
2010	* Performs lazy restoration of the set of host MSRs if they were previously
2011	* loaded with guest MSR values.
2012	*
2013	* @param pVCpu The cross context virtual CPU structure.
2014	*
2015	* @remarks No-long-jump zone!!!
2016	* @remarks The guest MSRs should have been saved back into the guest-CPU
2017	* context by hmR0VmxImportGuestState()!!!
2018	*/
2019	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2020	{
2021	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2022	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2023
2024	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2025	{
2026	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2027	#if HC_ARCH_BITS == 64
2028	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2029	{
2030	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2031	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2032	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2033	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2034	}
2035	#endif
2036	}
2037	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2038	}
2039
2040
2041	/**
2042	* Verifies that our cached values of the VMCS fields are all consistent with
2043	* what's actually present in the VMCS.
2044	*
2045	* @returns VBox status code.
2046	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2047	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2048	* VMCS content. HMCPU error-field is
2049	* updated, see VMX_VCI_XXX.
2050	* @param pVCpu The cross context virtual CPU structure.
2051	* @param pVmcsInfo The VMCS info. object.
2052	*/
2053	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2054	{
2055	uint32_t u32Val;
2056	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2057	AssertRCReturn(rc, rc);
2058	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2059	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32EntryCtls, u32Val),
2060	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2061	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2062
2063	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2064	AssertRCReturn(rc, rc);
2065	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2066	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ExitCtls, u32Val),
2067	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2068	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2069
2070	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2071	AssertRCReturn(rc, rc);
2072	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2073	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32PinCtls, u32Val),
2074	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2075	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2076
2077	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2078	AssertRCReturn(rc, rc);
2079	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2080	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls, u32Val),
2081	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2082	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2083
2084	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2085	{
2086	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2087	AssertRCReturn(rc, rc);
2088	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2089	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls2, u32Val),
2090	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2091	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2092	}
2093
2094	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2095	AssertRCReturn(rc, rc);
2096	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2097	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32XcptBitmap, u32Val),
2098	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2099	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2100
2101	uint64_t u64Val;
2102	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2103	AssertRCReturn(rc, rc);
2104	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2105	("Cache=%#RX64 VMCS=%#RX64\n", pVmcsInfo->u64TscOffset, u64Val),
2106	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2107	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2108
2109	return VINF_SUCCESS;
2110	}
2111
2112
2113	#ifdef VBOX_STRICT
2114	/**
2115	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2116	*
2117	* @param pVCpu The cross context virtual CPU structure.
2118	* @param pVmcsInfo The VMCS info. object.
2119	*/
2120	static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2121	{
2122	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2123
2124	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2125	{
2126	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2127	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2128	uint64_t uVmcsEferMsrVmcs;
2129	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2130	AssertRC(rc);
2131
2132	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2133	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2134	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2135	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2136	}
2137	}
2138
2139
2140	/**
2141	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2142	* VMCS are correct.
2143	*
2144	* @param pVCpu The cross context virtual CPU structure.
2145	* @param pVmcsInfo The VMCS info. object.
2146	*/
2147	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2148	{
2149	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2150
2151	/* Verify MSR counts in the VMCS are what we think it should be. */
2152	uint32_t cMsrs;
2153	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs);
2154	AssertRC(rc);
2155	Assert(cMsrs == pVmcsInfo->cEntryMsrLoad);
2156
2157	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs);
2158	AssertRC(rc);
2159	Assert(cMsrs == pVmcsInfo->cExitMsrStore);
2160
2161	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs);
2162	AssertRC(rc);
2163	Assert(cMsrs == pVmcsInfo->cExitMsrLoad);
2164
2165	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2166	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2167
2168	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2169	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2170	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2171	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2172	for (uint32_t i = 0; i < cMsrs; i++)
2173	{
2174	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2175	if (fSeparateExitMsrStorePage)
2176	{
2177	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2178	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2179	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2180	}
2181
2182	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2183	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2184	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2185
2186	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2187	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2188	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2189	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2190
2191	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2192	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2193	{
2194	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2195	if (pGuestMsrLoad->u32Msr == MSR_K6_EFER)
2196	{
2197	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2198	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2199	}
2200	else
2201	{
2202	AssertMsgReturnVoid((fMsrpm & (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR))
2203	== (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR),
2204	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2205	}
2206	}
2207
2208	/* Move to the next MSR. */
2209	pHostMsrLoad++;
2210	pGuestMsrLoad++;
2211	pGuestMsrStore++;
2212	}
2213	}
2214	#endif /* VBOX_STRICT */
2215
2216
2217	/**
2218	* Flushes the TLB using EPT.
2219	*
2220	* @returns VBox status code.
2221	* @param pVCpu The cross context virtual CPU structure of the calling
2222	* EMT. Can be NULL depending on @a enmTlbFlush.
2223	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2224	* enmTlbFlush.
2225	* @param enmTlbFlush Type of flush.
2226	*
2227	* @remarks Caller is responsible for making sure this function is called only
2228	* when NestedPaging is supported and providing @a enmTlbFlush that is
2229	* supported by the CPU.
2230	* @remarks Can be called with interrupts disabled.
2231	*/
2232	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2233	{
2234	uint64_t au64Descriptor[2];
2235	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2236	au64Descriptor[0] = 0;
2237	else
2238	{
2239	Assert(pVCpu);
2240	Assert(pVmcsInfo);
2241	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2242	}
2243	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2244
2245	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2246	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2247
2248	if ( RT_SUCCESS(rc)
2249	&& pVCpu)
2250	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2251	}
2252
2253
2254	/**
2255	* Flushes the TLB using VPID.
2256	*
2257	* @returns VBox status code.
2258	* @param pVCpu The cross context virtual CPU structure of the calling
2259	* EMT. Can be NULL depending on @a enmTlbFlush.
2260	* @param enmTlbFlush Type of flush.
2261	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2262	* on @a enmTlbFlush).
2263	*
2264	* @remarks Can be called with interrupts disabled.
2265	*/
2266	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2267	{
2268	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2269
2270	uint64_t au64Descriptor[2];
2271	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2272	{
2273	au64Descriptor[0] = 0;
2274	au64Descriptor[1] = 0;
2275	}
2276	else
2277	{
2278	AssertPtr(pVCpu);
2279	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2280	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2281	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2282	au64Descriptor[1] = GCPtr;
2283	}
2284
2285	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2286	AssertMsg(rc == VINF_SUCCESS,
2287	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2288
2289	if ( RT_SUCCESS(rc)
2290	&& pVCpu)
2291	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2292	NOREF(rc);
2293	}
2294
2295
2296	/**
2297	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2298	* otherwise there is nothing really to invalidate.
2299	*
2300	* @returns VBox status code.
2301	* @param pVCpu The cross context virtual CPU structure.
2302	* @param GCVirt Guest virtual address of the page to invalidate.
2303	*/
2304	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2305	{
2306	AssertPtr(pVCpu);
2307	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2308
2309	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2310	{
2311	/*
2312	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2313	* the EPT case. See @bugref{6043} and @bugref{6177}.
2314	*
2315	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2316	* as this function maybe called in a loop with individual addresses.
2317	*/
2318	PVM pVM = pVCpu->CTX_SUFF(pVM);
2319	if (pVM->hm.s.vmx.fVpid)
2320	{
2321	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2322
2323	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2324	/*
2325	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2326	* where executing INVVPID outside 64-bit mode does not flush translations of
2327	* 64-bit linear addresses, see @bugref{6208#c72}.
2328	*/
2329	if (RT_HI_U32(GCVirt))
2330	fVpidFlush = false;
2331	#endif
2332
2333	if (fVpidFlush)
2334	{
2335	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2336	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2337	}
2338	else
2339	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2340	}
2341	else if (pVM->hm.s.fNestedPaging)
2342	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2343	}
2344
2345	return VINF_SUCCESS;
2346	}
2347
2348
2349	/**
2350	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2351	* case where neither EPT nor VPID is supported by the CPU.
2352	*
2353	* @param pHostCpu The HM physical-CPU structure.
2354	* @param pVCpu The cross context virtual CPU structure.
2355	*
2356	* @remarks Called with interrupts disabled.
2357	*/
2358	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2359	{
2360	AssertPtr(pVCpu);
2361	AssertPtr(pHostCpu);
2362
2363	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2364
2365	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2366	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2367	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2368	pVCpu->hm.s.fForceTLBFlush = false;
2369	return;
2370	}
2371
2372
2373	/**
2374	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2375	*
2376	* @param pHostCpu The HM physical-CPU structure.
2377	* @param pVCpu The cross context virtual CPU structure.
2378	* @param pVmcsInfo The VMCS info. object.
2379	*
2380	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2381	* nomenclature. The reason is, to avoid confusion in compare statements
2382	* since the host-CPU copies are named "ASID".
2383	*
2384	* @remarks Called with interrupts disabled.
2385	*/
2386	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2387	{
2388	#ifdef VBOX_WITH_STATISTICS
2389	bool fTlbFlushed = false;
2390	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2391	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2392	if (!fTlbFlushed) \
2393	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2394	} while (0)
2395	#else
2396	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2397	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2398	#endif
2399
2400	AssertPtr(pVCpu);
2401	AssertPtr(pHostCpu);
2402	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2403
2404	PVM pVM = pVCpu->CTX_SUFF(pVM);
2405	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2406	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2407	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2408
2409	/*
2410	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2411	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2412	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2413	* cannot reuse the current ASID anymore.
2414	*/
2415	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2416	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2417	{
2418	++pHostCpu->uCurrentAsid;
2419	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2420	{
2421	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2422	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2423	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2424	}
2425
2426	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2427	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2428	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2429
2430	/*
2431	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2432	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2433	*/
2434	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2435	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2436	HMVMX_SET_TAGGED_TLB_FLUSHED();
2437	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2438	}
2439	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2440	{
2441	/*
2442	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2443	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2444	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2445	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2446	* mappings, see @bugref{6568}.
2447	*
2448	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2449	*/
2450	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2451	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2452	HMVMX_SET_TAGGED_TLB_FLUSHED();
2453	}
2454
2455	pVCpu->hm.s.fForceTLBFlush = false;
2456	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2457
2458	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
2459	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
2460	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2461	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2462	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2463	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2464	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2465	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2466	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2467
2468	/* Update VMCS with the VPID. */
2469	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2470	AssertRC(rc);
2471
2472	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2473	}
2474
2475
2476	/**
2477	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2478	*
2479	* @param pHostCpu The HM physical-CPU structure.
2480	* @param pVCpu The cross context virtual CPU structure.
2481	* @param pVmcsInfo The VMCS info. object.
2482	*
2483	* @remarks Called with interrupts disabled.
2484	*/
2485	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2486	{
2487	AssertPtr(pVCpu);
2488	AssertPtr(pHostCpu);
2489	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2490	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2491	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2492
2493	/*
2494	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2495	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2496	*/
2497	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2498	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2499	{
2500	pVCpu->hm.s.fForceTLBFlush = true;
2501	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2502	}
2503
2504	/* Check for explicit TLB flushes. */
2505	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2506	{
2507	pVCpu->hm.s.fForceTLBFlush = true;
2508	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2509	}
2510
2511	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2512	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2513
2514	if (pVCpu->hm.s.fForceTLBFlush)
2515	{
2516	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2517	pVCpu->hm.s.fForceTLBFlush = false;
2518	}
2519	}
2520
2521
2522	/**
2523	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2524	*
2525	* @param pHostCpu The HM physical-CPU structure.
2526	* @param pVCpu The cross context virtual CPU structure.
2527	*
2528	* @remarks Called with interrupts disabled.
2529	*/
2530	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2531	{
2532	AssertPtr(pVCpu);
2533	AssertPtr(pHostCpu);
2534	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2535	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2536	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2537
2538	/*
2539	* Force a TLB flush for the first world switch if the current CPU differs from the one we
2540	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2541	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2542	* cannot reuse the current ASID anymore.
2543	*/
2544	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2545	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2546	{
2547	pVCpu->hm.s.fForceTLBFlush = true;
2548	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2549	}
2550
2551	/* Check for explicit TLB flushes. */
2552	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2553	{
2554	/*
2555	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2556	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2557	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
2558	* include fExplicitFlush's too) - an obscure corner case.
2559	*/
2560	pVCpu->hm.s.fForceTLBFlush = true;
2561	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2562	}
2563
2564	PVM pVM = pVCpu->CTX_SUFF(pVM);
2565	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2566	if (pVCpu->hm.s.fForceTLBFlush)
2567	{
2568	++pHostCpu->uCurrentAsid;
2569	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2570	{
2571	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2572	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2573	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2574	}
2575
2576	pVCpu->hm.s.fForceTLBFlush = false;
2577	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2578	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2579	if (pHostCpu->fFlushAsidBeforeUse)
2580	{
2581	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2582	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2583	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2584	{
2585	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2586	pHostCpu->fFlushAsidBeforeUse = false;
2587	}
2588	else
2589	{
2590	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2591	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2592	}
2593	}
2594	}
2595
2596	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2597	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2598	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2599	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2600	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2601	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2602	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2603
2604	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2605	AssertRC(rc);
2606	}
2607
2608
2609	/**
2610	* Flushes the guest TLB entry based on CPU capabilities.
2611	*
2612	* @param pHostCpu The HM physical-CPU structure.
2613	* @param pVCpu The cross context virtual CPU structure.
2614	* @param pVmcsInfo The VMCS info. object.
2615	*
2616	* @remarks Called with interrupts disabled.
2617	*/
2618	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2619	{
2620	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2621	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2622	#endif
2623	PVM pVM = pVCpu->CTX_SUFF(pVM);
2624	switch (pVM->hm.s.vmx.enmTlbFlushType)
2625	{
2626	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
2627	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
2628	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
2629	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
2630	default:
2631	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2632	break;
2633	}
2634	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2635	}
2636
2637
2638	/**
2639	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2640	* TLB entries from the host TLB before VM-entry.
2641	*
2642	* @returns VBox status code.
2643	* @param pVM The cross context VM structure.
2644	*/
2645	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2646	{
2647	/*
2648	* Determine optimal flush type for nested paging.
2649	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
2650	* unrestricted guest execution (see hmR3InitFinalizeR0()).
2651	*/
2652	if (pVM->hm.s.fNestedPaging)
2653	{
2654	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2655	{
2656	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2657	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2658	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2659	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2660	else
2661	{
2662	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2663	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2664	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2665	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2666	}
2667
2668	/* Make sure the write-back cacheable memory type for EPT is supported. */
2669	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2670	{
2671	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2672	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2673	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2674	}
2675
2676	/* EPT requires a page-walk length of 4. */
2677	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2678	{
2679	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2680	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2681	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2682	}
2683	}
2684	else
2685	{
2686	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2687	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2688	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2689	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2690	}
2691	}
2692
2693	/*
2694	* Determine optimal flush type for VPID.
2695	*/
2696	if (pVM->hm.s.vmx.fVpid)
2697	{
2698	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2699	{
2700	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2701	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2702	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2703	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2704	else
2705	{
2706	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2707	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2708	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2709	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2710	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2711	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2712	pVM->hm.s.vmx.fVpid = false;
2713	}
2714	}
2715	else
2716	{
2717	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2718	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2719	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2720	pVM->hm.s.vmx.fVpid = false;
2721	}
2722	}
2723
2724	/*
2725	* Setup the handler for flushing tagged-TLBs.
2726	*/
2727	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2728	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2729	else if (pVM->hm.s.fNestedPaging)
2730	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2731	else if (pVM->hm.s.vmx.fVpid)
2732	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2733	else
2734	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2735	return VINF_SUCCESS;
2736	}
2737
2738
2739	/**
2740	* Sets up the virtual-APIC page address for the VMCS.
2741	*
2742	* @returns VBox status code.
2743	* @param pVCpu The cross context virtual CPU structure.
2744	* @param pVmcsInfo The VMCS info. object.
2745	*/
2746	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2747	{
2748	NOREF(pVCpu);
2749	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
2750	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
2751	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2752	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
2753	}
2754
2755
2756	/**
2757	* Sets up the MSR-bitmap address for the VMCS.
2758	*
2759	* @returns VBox status code.
2760	* @param pVCpu The cross context virtual CPU structure.
2761	* @param pVmcsInfo The VMCS info. object.
2762	*/
2763	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2764	{
2765	NOREF(pVCpu);
2766	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
2767	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
2768	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2769	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
2770	}
2771
2772
2773	/**
2774	* Sets up the APIC-access page address for the VMCS.
2775	*
2776	* @returns VBox status code.
2777	* @param pVCpu The cross context virtual CPU structure.
2778	*/
2779	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
2780	{
2781	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
2782	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
2783	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2784	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
2785	}
2786
2787
2788	/**
2789	* Sets up the VMCS link pointer for the VMCS.
2790	*
2791	* @returns VBox status code.
2792	* @param pVCpu The cross context virtual CPU structure.
2793	* @param pVmcsInfo The VMCS info. object.
2794	*/
2795	DECLINLINE(int) hmR0VmxSetupVmcsLinkPtr(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2796	{
2797	NOREF(pVCpu);
2798	uint64_t const u64VmcsLinkPtr = pVmcsInfo->u64VmcsLinkPtr;
2799	Assert(u64VmcsLinkPtr == UINT64_C(0xffffffffffffffff)); /* Bits 63:0 MB1. */
2800	return VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, u64VmcsLinkPtr);
2801	}
2802
2803
2804	/**
2805	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
2806	* in the VMCS.
2807	*
2808	* @returns VBox status code.
2809	* @param pVCpu The cross context virtual CPU structure.
2810	* @param pVmcsInfo The VMCS info. object.
2811	*/
2812	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2813	{
2814	NOREF(pVCpu);
2815	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
2816	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
2817	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2818
2819	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
2820	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
2821	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
2822
2823	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
2824	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
2825	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2826
2827	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
2828	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
2829	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
2830	AssertRCReturn(rc, rc);
2831	return VINF_SUCCESS;
2832	}
2833
2834
2835	/**
2836	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
2837	*
2838	* @param pVCpu The cross context virtual CPU structure.
2839	* @param pVmcsInfo The VMCS info. object.
2840	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2841	*/
2842	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2843	{
2844	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
2845
2846	/*
2847	* The guest can access the following MSRs (read, write) without causing
2848	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
2849	*/
2850	PVM pVM = pVCpu->CTX_SUFF(pVM);
2851	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2852	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2853	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2854	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2855	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2856
2857	#ifdef VBOX_STRICT
2858	/** @todo NSTVMX: Remove this later. */
2859	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_IA32_SYSENTER_CS);
2860	Assert((fMsrpm & (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR)) == (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR));
2861
2862	fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K8_GS_BASE);
2863	Assert((fMsrpm & (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR)) == (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR));
2864	#endif
2865
2866	/*
2867	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
2868	* associated with then. We never need to intercept access (writes need to be
2869	* executed without causing a VM-exit, reads will #GP fault anyway).
2870	*
2871	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
2872	* read/write them. We swap the the guest/host MSR value using the
2873	* auto-load/store MSR area.
2874	*/
2875	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2876	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2877	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
2878	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2879	if (pVM->cpum.ro.GuestFeatures.fIbrs)
2880	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2881
2882	/*
2883	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
2884	*/
2885
2886	#if HC_ARCH_BITS == 64
2887	/*
2888	* Allow full read/write access for the following MSRs (mandatory for VT-x)
2889	* required for 64-bit guests.
2890	*/
2891	if (pVM->hm.s.fAllow64BitGuests)
2892	{
2893	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2894	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K6_STAR, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2895	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2896	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2897
2898	# ifdef VBOX_STRICT
2899	fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K8_GS_BASE);
2900	Assert((fMsrpm & (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR)) == (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR));
2901	# endif
2902	}
2903	#endif
2904	}
2905
2906
2907	/**
2908	* Sets up pin-based VM-execution controls in the VMCS.
2909	*
2910	* @returns VBox status code.
2911	* @param pVCpu The cross context virtual CPU structure.
2912	* @param pVmcsInfo The VMCS info. object.
2913	*/
2914	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2915	{
2916	PVM pVM = pVCpu->CTX_SUFF(pVM);
2917	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
2918	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2919
2920	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2921	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2922
2923	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2924	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2925
2926	/* Enable the VMX-preemption timer. */
2927	if (pVM->hm.s.vmx.fUsePreemptTimer)
2928	{
2929	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2930	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
2931	}
2932
2933	#if 0
2934	/* Enable posted-interrupt processing. */
2935	if (pVM->hm.s.fPostedIntrs)
2936	{
2937	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2938	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2939	fVal \|= VMX_PIN_CTL_POSTED_INT;
2940	}
2941	#endif
2942
2943	if ((fVal & fZap) != fVal)
2944	{
2945	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2946	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
2947	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2948	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2949	}
2950
2951	/* Commit it to the VMCS and update our cache. */
2952	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2953	AssertRCReturn(rc, rc);
2954	pVmcsInfo->u32PinCtls = fVal;
2955
2956	return VINF_SUCCESS;
2957	}
2958
2959
2960	/**
2961	* Sets up secondary processor-based VM-execution controls in the VMCS.
2962	*
2963	* @returns VBox status code.
2964	* @param pVCpu The cross context virtual CPU structure.
2965	* @param pVmcsInfo The VMCS info. object.
2966	*/
2967	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2968	{
2969	PVM pVM = pVCpu->CTX_SUFF(pVM);
2970	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
2971	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2972
2973	/* WBINVD causes a VM-exit. */
2974	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2975	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
2976
2977	/* Enable EPT (aka nested-paging). */
2978	if (pVM->hm.s.fNestedPaging)
2979	fVal \|= VMX_PROC_CTLS2_EPT;
2980
2981	/* Enable the INVPCID instruction if supported by the hardware and we expose
2982	it to the guest. Without this, guest executing INVPCID would cause a #UD. */
2983	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2984	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
2985	fVal \|= VMX_PROC_CTLS2_INVPCID;
2986
2987	/* Enable VPID. */
2988	if (pVM->hm.s.vmx.fVpid)
2989	fVal \|= VMX_PROC_CTLS2_VPID;
2990
2991	/* Enable unrestricted guest execution. */
2992	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2993	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
2994
2995	#if 0
2996	if (pVM->hm.s.fVirtApicRegs)
2997	{
2998	/* Enable APIC-register virtualization. */
2999	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3000	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3001
3002	/* Enable virtual-interrupt delivery. */
3003	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3004	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3005	}
3006	#endif
3007
3008	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
3009	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3010	* done dynamically. */
3011	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3012	{
3013	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3014	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3015	AssertRCReturn(rc, rc);
3016	}
3017
3018	/* Enable the RDTSCP instruction if supported by the hardware and we expose
3019	it to the guest. Without this, guest executing RDTSCP would cause a #UD. */
3020	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
3021	&& pVM->cpum.ro.GuestFeatures.fRdTscP)
3022	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3023
3024	/* Enable Pause-Loop exiting. */
3025	if ( pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT
3026	&& pVM->hm.s.vmx.cPleGapTicks
3027	&& pVM->hm.s.vmx.cPleWindowTicks)
3028	{
3029	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3030
3031	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3032	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3033	AssertRCReturn(rc, rc);
3034	}
3035
3036	if ((fVal & fZap) != fVal)
3037	{
3038	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3039	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3040	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3041	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3042	}
3043
3044	/* Commit it to the VMCS and update our cache. */
3045	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3046	AssertRCReturn(rc, rc);
3047	pVmcsInfo->u32ProcCtls2 = fVal;
3048
3049	return VINF_SUCCESS;
3050	}
3051
3052
3053	/**
3054	* Sets up processor-based VM-execution controls in the VMCS.
3055	*
3056	* @returns VBox status code.
3057	* @param pVCpu The cross context virtual CPU structure.
3058	* @param pVmcsInfo The VMCS info. object.
3059	*/
3060	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3061	{
3062	PVM pVM = pVCpu->CTX_SUFF(pVM);
3063
3064	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3065	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3066
3067	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3068	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3069	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3070	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3071	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3072	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3073	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3074
3075	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3076	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3077	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3078	{
3079	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3080	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3081	}
3082
3083	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3084	if (!pVM->hm.s.fNestedPaging)
3085	{
3086	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3087	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3088	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3089	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3090	}
3091
3092	/* Use TPR shadowing if supported by the CPU. */
3093	if ( PDMHasApic(pVM)
3094	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3095	{
3096	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3097	/* CR8 writes cause a VM-exit based on TPR threshold. */
3098	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3099	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3100	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3101	AssertRCReturn(rc, rc);
3102	}
3103	else
3104	{
3105	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3106	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3107	if (pVM->hm.s.fAllow64BitGuests)
3108	{
3109	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3110	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3111	}
3112	}
3113
3114	/* Use MSR-bitmaps if supported by the CPU. */
3115	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3116	{
3117	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3118	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3119	AssertRCReturn(rc, rc);
3120	}
3121
3122	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3123	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3124	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3125
3126	if ((fVal & fZap) != fVal)
3127	{
3128	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3129	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3130	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3131	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3132	}
3133
3134	/* Commit it to the VMCS and update our cache. */
3135	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3136	AssertRCReturn(rc, rc);
3137	pVmcsInfo->u32ProcCtls = fVal;
3138
3139	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3140	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo, false /* fIsNstGstVmcs */);
3141
3142	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3143	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3144	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3145
3146	/* Sanity check, should not really happen. */
3147	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3148	{ /* likely */ }
3149	else
3150	{
3151	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3152	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3153	}
3154
3155	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3156	return VINF_SUCCESS;
3157	}
3158
3159
3160	/**
3161	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3162	* Processor-based VM-execution) control fields in the VMCS.
3163	*
3164	* @returns VBox status code.
3165	* @param pVCpu The cross context virtual CPU structure.
3166	* @param pVmcsInfo The VMCS info. object.
3167	*/
3168	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3169	{
3170	/* Set the auto-load/store MSR area addresses in the VMCS. */
3171	int rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3172	if (RT_SUCCESS(rc))
3173	{
3174	/* Set the VMCS link pointer in the VMCS. */
3175	rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3176	if (RT_SUCCESS(rc))
3177	{
3178	/* Set the CR0/CR4 guest/host mask. */
3179	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3180	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3181	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3182	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3183	if (RT_SUCCESS(rc))
3184	{
3185	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3186	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3187	return VINF_SUCCESS;
3188	}
3189	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3190	}
3191	else
3192	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3193	}
3194	else
3195	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3196	return rc;
3197	}
3198
3199
3200	/**
3201	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3202	*
3203	* We shall setup those exception intercepts that don't change during the
3204	* lifetime of the VM here. The rest are done dynamically while loading the
3205	* guest state.
3206	*
3207	* @returns VBox status code.
3208	* @param pVCpu The cross context virtual CPU structure.
3209	* @param pVmcsInfo The VMCS info. object.
3210	*/
3211	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3212	{
3213	/*
3214	* The following exceptions are always intercepted:
3215	*
3216	* #AC - To prevent the guest from hanging the CPU.
3217	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3218	* recursive #DBs can cause a CPU hang.
3219	* #PF - To sync our shadow page tables when nested-paging is not used.
3220	*/
3221	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3222	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3223	\| RT_BIT(X86_XCPT_DB)
3224	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3225
3226	/* Commit it to the VMCS. */
3227	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3228	AssertRCReturn(rc, rc);
3229
3230	/* Update our cache of the exception bitmap. */
3231	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3232	return VINF_SUCCESS;
3233	}
3234
3235
3236	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3237	/**
3238	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3239	*
3240	* @returns VBox status code.
3241	* @param pVCpu The cross context virtual CPU structure.
3242	* @param pVmcsInfo The VMCS info. object.
3243	*/
3244	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3245	{
3246	PVM pVM = pVCpu->CTX_SUFF(pVM);
3247	int rc = hmR0VmxSetupVmcsLinkPtr(pVmcsInfo);
3248	if (RT_SUCCESS(rc))
3249	{
3250	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
3251	if (RT_SUCCESS(rc))
3252	{
3253	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3254	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
3255	if (RT_SUCCESS(rc))
3256	{
3257	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3258	rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3259	if (RT_SUCCESS(rc))
3260	return VINF_SUCCESS;
3261
3262	LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3263	}
3264	else
3265	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3266	}
3267	else
3268	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3269	}
3270	else
3271	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3272
3273	return rc;
3274	}
3275	#endif
3276
3277
3278	/**
3279	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3280	* VMX.
3281	*
3282	* @returns VBox status code.
3283	* @param pVCpu The cross context virtual CPU structure.
3284	* @param pVmcsInfo The VMCS info. object.
3285	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3286	*/
3287	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3288	{
3289	Assert(pVmcsInfo);
3290	Assert(pVmcsInfo->pvVmcs);
3291	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3292
3293	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3294	PVM pVM = pVCpu->CTX_SUFF(pVM);
3295	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3296	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3297
3298	LogFlowFunc(("\n"));
3299
3300	/*
3301	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3302	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3303	*/
3304	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3305	if (RT_SUCCESS(rc))
3306	{
3307	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3308	if (RT_SUCCESS(rc))
3309	{
3310	if (!fIsNstGstVmcs)
3311	{
3312	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3313	if (RT_SUCCESS(rc))
3314	{
3315	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
3316	if (RT_SUCCESS(rc))
3317	{
3318	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
3319	if (RT_SUCCESS(rc))
3320	{
3321	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
3322	if (RT_SUCCESS(rc))
3323	{ /* likely */ }
3324	else
3325	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
3326	}
3327	else
3328	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
3329	}
3330	else
3331	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
3332	}
3333	else
3334	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
3335	}
3336	else
3337	{
3338	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3339	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
3340	if (RT_SUCCESS(rc))
3341	{ /* likely */ }
3342	else
3343	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
3344	#else
3345	AssertFailed();
3346	#endif
3347	}
3348	}
3349	else
3350	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
3351	}
3352	else
3353	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
3354
3355	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
3356	if (RT_SUCCESS(rc))
3357	{
3358	rc = hmR0VmxClearVmcs(pVmcsInfo);
3359	if (RT_SUCCESS(rc))
3360	{ /* likely */ }
3361	else
3362	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
3363	}
3364
3365	/*
3366	* Update the last-error record both for failures and success, so we
3367	* can propagate the status code back to ring-3 for diagnostics.
3368	*/
3369	hmR0VmxUpdateErrorRecord(pVCpu, rc);
3370	NOREF(pszVmcs);
3371	return rc;
3372	}
3373
3374
3375	/**
3376	* Does global VT-x initialization (called during module initialization).
3377	*
3378	* @returns VBox status code.
3379	*/
3380	VMMR0DECL(int) VMXR0GlobalInit(void)
3381	{
3382	#ifdef HMVMX_USE_FUNCTION_TABLE
3383	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
3384	# ifdef VBOX_STRICT
3385	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
3386	Assert(g_apfnVMExitHandlers[i]);
3387	# endif
3388	#endif
3389	return VINF_SUCCESS;
3390	}
3391
3392
3393	/**
3394	* Does global VT-x termination (called during module termination).
3395	*/
3396	VMMR0DECL(void) VMXR0GlobalTerm()
3397	{
3398	/* Nothing to do currently. */
3399	}
3400
3401
3402	/**
3403	* Sets up and activates VT-x on the current CPU.
3404	*
3405	* @returns VBox status code.
3406	* @param pHostCpu The HM physical-CPU structure.
3407	* @param pVM The cross context VM structure. Can be
3408	* NULL after a host resume operation.
3409	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
3410	* fEnabledByHost is @c true).
3411	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
3412	* @a fEnabledByHost is @c true).
3413	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
3414	* enable VT-x on the host.
3415	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
3416	*/
3417	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
3418	PCSUPHWVIRTMSRS pHwvirtMsrs)
3419	{
3420	Assert(pHostCpu);
3421	Assert(pHwvirtMsrs);
3422	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3423
3424	/* Enable VT-x if it's not already enabled by the host. */
3425	if (!fEnabledByHost)
3426	{
3427	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
3428	if (RT_FAILURE(rc))
3429	return rc;
3430	}
3431
3432	/*
3433	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
3434	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
3435	* invalidated when flushing by VPID.
3436	*/
3437	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3438	{
3439	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
3440	pHostCpu->fFlushAsidBeforeUse = false;
3441	}
3442	else
3443	pHostCpu->fFlushAsidBeforeUse = true;
3444
3445	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
3446	++pHostCpu->cTlbFlushes;
3447
3448	return VINF_SUCCESS;
3449	}
3450
3451
3452	/**
3453	* Deactivates VT-x on the current CPU.
3454	*
3455	* @returns VBox status code.
3456	* @param pvCpuPage Pointer to the VMXON region.
3457	* @param HCPhysCpuPage Physical address of the VMXON region.
3458	*
3459	* @remarks This function should never be called when SUPR0EnableVTx() or
3460	* similar was used to enable VT-x on the host.
3461	*/
3462	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
3463	{
3464	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
3465
3466	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3467	return hmR0VmxLeaveRootMode();
3468	}
3469
3470
3471	/**
3472	* Does per-VM VT-x initialization.
3473	*
3474	* @returns VBox status code.
3475	* @param pVM The cross context VM structure.
3476	*/
3477	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
3478	{
3479	LogFlowFunc(("pVM=%p\n", pVM));
3480
3481	int rc = hmR0VmxStructsAlloc(pVM);
3482	if (RT_FAILURE(rc))
3483	{
3484	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
3485	return rc;
3486	}
3487
3488	return VINF_SUCCESS;
3489	}
3490
3491
3492	/**
3493	* Does per-VM VT-x termination.
3494	*
3495	* @returns VBox status code.
3496	* @param pVM The cross context VM structure.
3497	*/
3498	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
3499	{
3500	LogFlowFunc(("pVM=%p\n", pVM));
3501
3502	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3503	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
3504	{
3505	Assert(pVM->hm.s.vmx.pvScratch);
3506	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
3507	}
3508	#endif
3509	hmR0VmxStructsFree(pVM);
3510	return VINF_SUCCESS;
3511	}
3512
3513
3514	/**
3515	* Sets up the VM for execution using hardware-assisted VMX.
3516	* This function is only called once per-VM during initialization.
3517	*
3518	* @returns VBox status code.
3519	* @param pVM The cross context VM structure.
3520	*/
3521	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
3522	{
3523	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
3524	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3525
3526	LogFlowFunc(("pVM=%p\n", pVM));
3527
3528	/*
3529	* At least verify if VMX is enabled, since we can't check if we're in
3530	* VMX root mode or not without causing a #GP.
3531	*/
3532	RTCCUINTREG const uHostCR4 = ASMGetCR4();
3533	if (RT_LIKELY(uHostCR4 & X86_CR4_VMXE))
3534	{ /* likely */ }
3535	else
3536	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
3537
3538	/*
3539	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
3540	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
3541	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
3542	*/
3543	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3544	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
3545	\|\| !pVM->hm.s.vmx.pRealModeTSS))
3546	{
3547	LogRelFunc(("Invalid real-on-v86 state.\n"));
3548	return VERR_INTERNAL_ERROR;
3549	}
3550
3551	/* Initialize these always, see hmR3InitFinalizeR0().*/
3552	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
3553	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
3554
3555	/* Setup the tagged-TLB flush handlers. */
3556	int rc = hmR0VmxSetupTaggedTlb(pVM);
3557	if (RT_FAILURE(rc))
3558	{
3559	LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
3560	return rc;
3561	}
3562
3563	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
3564	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
3565	#if HC_ARCH_BITS == 64
3566	if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
3567	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
3568	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
3569	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
3570	#endif
3571
3572	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3573	{
3574	PVMCPU pVCpu = &pVM->aCpus[idCpu];
3575	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
3576
3577	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
3578	if (RT_SUCCESS(rc))
3579	{
3580	#if HC_ARCH_BITS == 32
3581	hmR0VmxInitVmcsReadCache(pVCpu);
3582	#endif
3583	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3584	if (pVM->cpum.ro.GuestFeatures.fVmx)
3585	{
3586	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
3587	if (RT_SUCCESS(rc))
3588	{ /* likely */ }
3589	else
3590	{
3591	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
3592	return rc;
3593	}
3594	}
3595	#endif
3596	}
3597	else
3598	{
3599	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
3600	return rc;
3601	}
3602	}
3603
3604	return VINF_SUCCESS;
3605	}
3606
3607
3608	#if HC_ARCH_BITS == 32
3609	# ifdef VBOX_ENABLE_64_BITS_GUESTS
3610	/**
3611	* Check if guest state allows safe use of 32-bit switcher again.
3612	*
3613	* Segment bases and protected mode structures must be 32-bit addressable
3614	* because the 32-bit switcher will ignore high dword when writing these VMCS
3615	* fields. See @bugref{8432} for details.
3616	*
3617	* @returns true if safe, false if must continue to use the 64-bit switcher.
3618	* @param pCtx Pointer to the guest-CPU context.
3619	*
3620	* @remarks No-long-jump zone!!!
3621	*/
3622	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
3623	{
3624	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
3625	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
3626	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3627	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3628	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
3629	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3630	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
3631	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
3632	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3633	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3634
3635	/* All good, bases are 32-bit. */
3636	return true;
3637	}
3638	# endif /* VBOX_ENABLE_64_BITS_GUESTS */
3639
3640	# ifdef VBOX_STRICT
3641	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
3642	{
3643	switch (idxField)
3644	{
3645	case VMX_VMCS_GUEST_RIP:
3646	case VMX_VMCS_GUEST_RSP:
3647	case VMX_VMCS_GUEST_SYSENTER_EIP:
3648	case VMX_VMCS_GUEST_SYSENTER_ESP:
3649	case VMX_VMCS_GUEST_GDTR_BASE:
3650	case VMX_VMCS_GUEST_IDTR_BASE:
3651	case VMX_VMCS_GUEST_CS_BASE:
3652	case VMX_VMCS_GUEST_DS_BASE:
3653	case VMX_VMCS_GUEST_ES_BASE:
3654	case VMX_VMCS_GUEST_FS_BASE:
3655	case VMX_VMCS_GUEST_GS_BASE:
3656	case VMX_VMCS_GUEST_SS_BASE:
3657	case VMX_VMCS_GUEST_LDTR_BASE:
3658	case VMX_VMCS_GUEST_TR_BASE:
3659	case VMX_VMCS_GUEST_CR3:
3660	return true;
3661	}
3662	return false;
3663	}
3664
3665	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3666	{
3667	switch (idxField)
3668	{
3669	/* Read-only fields. */
3670	case VMX_VMCS_RO_EXIT_QUALIFICATION:
3671	return true;
3672	}
3673	/* Remaining readable fields should also be writable. */
3674	return hmR0VmxIsValidWriteField(idxField);
3675	}
3676	# endif /* VBOX_STRICT */
3677
3678
3679	/**
3680	* Executes the specified handler in 64-bit mode.
3681	*
3682	* @returns VBox status code (no informational status codes).
3683	* @param pVCpu The cross context virtual CPU structure.
3684	* @param enmOp The operation to perform.
3685	* @param cParams Number of parameters.
3686	* @param paParam Array of 32-bit parameters.
3687	*/
3688	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
3689	{
3690	PVM pVM = pVCpu->CTX_SUFF(pVM);
3691	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
3692	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
3693	Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
3694	Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
3695
3696	#ifdef VBOX_STRICT
3697	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
3698	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
3699
3700	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
3701	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
3702	#endif
3703
3704	/* Disable interrupts. */
3705	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
3706
3707	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
3708	RTCPUID idHostCpu = RTMpCpuId();
3709	CPUMR0SetLApic(pVCpu, idHostCpu);
3710	#endif
3711
3712	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
3713
3714	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3715	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3716
3717	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
3718	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3719	hmR0VmxClearVmcs(pVmcsInfo);
3720
3721	/* Leave VMX root mode and disable VMX. */
3722	VMXDisable();
3723	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3724
3725	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
3726	CPUMSetHyperEIP(pVCpu, enmOp);
3727	for (int i = (int)cParams - 1; i >= 0; i--)
3728	CPUMPushHyper(pVCpu, paParam[i]);
3729
3730	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
3731
3732	/* Call the switcher. */
3733	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
3734	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
3735
3736	/* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
3737	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
3738
3739	/* Re-enter VMX root mode. */
3740	int rc2 = VMXEnable(HCPhysCpuPage);
3741	if (RT_FAILURE(rc2))
3742	{
3743	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3744	ASMSetFlags(fOldEFlags);
3745	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
3746	return rc2;
3747	}
3748
3749	/* Restore the VMCS as the current VMCS. */
3750	rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
3751	AssertRC(rc2);
3752	Assert(!(ASMGetFlags() & X86_EFL_IF));
3753	ASMSetFlags(fOldEFlags);
3754	return rc;
3755	}
3756
3757
3758	/**
3759	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
3760	* supporting 64-bit guests.
3761	*
3762	* @returns VBox status code.
3763	* @param fResume Whether to VMLAUNCH or VMRESUME.
3764	* @param pCtx Pointer to the guest-CPU context.
3765	* @param pCache Pointer to the VMCS batch cache.
3766	* @param pVM The cross context VM structure.
3767	* @param pVCpu The cross context virtual CPU structure.
3768	*/
3769	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
3770	{
3771	NOREF(fResume);
3772
3773	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3774	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3775	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3776
3777	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3778	pCache->uPos = 1;
3779	pCache->interPD = PGMGetInterPaeCR3(pVM);
3780	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
3781	#endif
3782
3783	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3784	pCache->TestIn.HCPhysCpuPage = 0;
3785	pCache->TestIn.HCPhysVmcs = 0;
3786	pCache->TestIn.pCache = 0;
3787	pCache->TestOut.HCPhysVmcs = 0;
3788	pCache->TestOut.pCache = 0;
3789	pCache->TestOut.pCtx = 0;
3790	pCache->TestOut.eflags = 0;
3791	#else
3792	NOREF(pCache);
3793	#endif
3794
3795	uint32_t aParam[10];
3796	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
3797	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
3798	aParam[2] = RT_LO_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
3799	aParam[3] = RT_HI_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
3800	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
3801	aParam[5] = 0;
3802	aParam[6] = VM_RC_ADDR(pVM, pVM);
3803	aParam[7] = 0;
3804	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
3805	aParam[9] = 0;
3806
3807	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3808	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
3809	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
3810	#endif
3811	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
3812
3813	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3814	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
3815	Assert(pCtx->dr[4] == 10);
3816	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
3817	#endif
3818
3819	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3820	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
3821	AssertMsg(pCache->TestIn.HCPhysVmcs == pVmcsInfo->HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3822	pVmcsInfo->HCPhysVmcs));
3823	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3824	pCache->TestOut.HCPhysVmcs));
3825	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
3826	pCache->TestOut.pCache));
3827	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
3828	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
3829	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
3830	pCache->TestOut.pCtx));
3831	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
3832	#endif
3833	NOREF(pCtx);
3834	return rc;
3835	}
3836	#endif
3837
3838
3839	/**
3840	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
3841	* the VMCS.
3842	*
3843	* @returns VBox status code.
3844	*/
3845	static int hmR0VmxExportHostControlRegs(void)
3846	{
3847	RTCCUINTREG uReg = ASMGetCR0();
3848	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
3849	AssertRCReturn(rc, rc);
3850
3851	uReg = ASMGetCR3();
3852	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
3853	AssertRCReturn(rc, rc);
3854
3855	uReg = ASMGetCR4();
3856	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
3857	AssertRCReturn(rc, rc);
3858	return rc;
3859	}
3860
3861
3862	/**
3863	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
3864	* the host-state area in the VMCS.
3865	*
3866	* @returns VBox status code.
3867	* @param pVCpu The cross context virtual CPU structure.
3868	*/
3869	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
3870	{
3871	#if HC_ARCH_BITS == 64
3872	/**
3873	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
3874	* requirements. See hmR0VmxExportHostSegmentRegs().
3875	*/
3876	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
3877	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
3878	{ \
3879	bool fValidSelector = true; \
3880	if ((selValue) & X86_SEL_LDT) \
3881	{ \
3882	uint32_t uAttr = ASMGetSegAttr((selValue)); \
3883	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
3884	} \
3885	if (fValidSelector) \
3886	{ \
3887	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
3888	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
3889	} \
3890	(selValue) = 0; \
3891	}
3892
3893	/*
3894	* If we've executed guest code using hardware-assisted VMX, the host-state bits
3895	* will be messed up. We should -not- save the messed up state without restoring
3896	* the original host-state, see @bugref{7240}.
3897	*
3898	* This apparently can happen (most likely the FPU changes), deal with it rather than
3899	* asserting. Was observed booting Solaris 10u10 32-bit guest.
3900	*/
3901	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
3902	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
3903	{
3904	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
3905	pVCpu->idCpu));
3906	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
3907	}
3908	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
3909	#else
3910	RT_NOREF(pVCpu);
3911	#endif
3912
3913	/*
3914	* Host DS, ES, FS and GS segment registers.
3915	*/
3916	#if HC_ARCH_BITS == 64
3917	RTSEL uSelDS = ASMGetDS();
3918	RTSEL uSelES = ASMGetES();
3919	RTSEL uSelFS = ASMGetFS();
3920	RTSEL uSelGS = ASMGetGS();
3921	#else
3922	RTSEL uSelDS = 0;
3923	RTSEL uSelES = 0;
3924	RTSEL uSelFS = 0;
3925	RTSEL uSelGS = 0;
3926	#endif
3927
3928	/*
3929	* Host CS and SS segment registers.
3930	*/
3931	RTSEL uSelCS = ASMGetCS();
3932	RTSEL uSelSS = ASMGetSS();
3933
3934	/*
3935	* Host TR segment register.
3936	*/
3937	RTSEL uSelTR = ASMGetTR();
3938
3939	#if HC_ARCH_BITS == 64
3940	/*
3941	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
3942	* gain VM-entry and restore them before we get preempted.
3943	*
3944	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
3945	*/
3946	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3947	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3948	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3949	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3950	# undef VMXLOCAL_ADJUST_HOST_SEG
3951	#endif
3952
3953	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
3954	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
3955	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
3956	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
3957	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
3958	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
3959	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
3960	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
3961	Assert(uSelCS);
3962	Assert(uSelTR);
3963
3964	/* Write these host selector fields into the host-state area in the VMCS. */
3965	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3966	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3967	#if HC_ARCH_BITS == 64
3968	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3969	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3970	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3971	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3972	#else
3973	NOREF(uSelDS);
3974	NOREF(uSelES);
3975	NOREF(uSelFS);
3976	NOREF(uSelGS);
3977	#endif
3978	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3979	AssertRCReturn(rc, rc);
3980
3981	/*
3982	* Host GDTR and IDTR.
3983	*/
3984	RTGDTR Gdtr;
3985	RTIDTR Idtr;
3986	RT_ZERO(Gdtr);
3987	RT_ZERO(Idtr);
3988	ASMGetGDTR(&Gdtr);
3989	ASMGetIDTR(&Idtr);
3990	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3991	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3992	AssertRCReturn(rc, rc);
3993
3994	#if HC_ARCH_BITS == 64
3995	/*
3996	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
3997	* them to the maximum limit (0xffff) on every VM-exit.
3998	*/
3999	if (Gdtr.cbGdt != 0xffff)
4000	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4001
4002	/*
4003	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4004	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4005	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4006	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4007	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4008	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4009	* at 0xffff on hosts where we are sure it won't cause trouble.
4010	*/
4011	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4012	if (Idtr.cbIdt < 0x0fff)
4013	# else
4014	if (Idtr.cbIdt != 0xffff)
4015	# endif
4016	{
4017	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4018	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4019	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4020	}
4021	#endif
4022
4023	/*
4024	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4025	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4026	* RPL should be too in most cases.
4027	*/
4028	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4029	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4030
4031	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4032	#if HC_ARCH_BITS == 64
4033	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4034
4035	/*
4036	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4037	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4038	* restoration if the host has something else. Task switching is not supported in 64-bit
4039	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4040	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4041	*
4042	* [1] See Intel spec. 3.5 "System Descriptor Types".
4043	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4044	*/
4045	PVM pVM = pVCpu->CTX_SUFF(pVM);
4046	Assert(pDesc->System.u4Type == 11);
4047	if ( pDesc->System.u16LimitLow != 0x67
4048	\|\| pDesc->System.u4LimitHigh)
4049	{
4050	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4051	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4052	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4053	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4054	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4055	}
4056
4057	/*
4058	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4059	*/
4060	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4061	{
4062	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4063	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4064	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4065	{
4066	/* The GDT is read-only but the writable GDT is available. */
4067	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4068	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4069	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4070	AssertRCReturn(rc, rc);
4071	}
4072	}
4073	#else
4074	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4075	#endif
4076	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4077	AssertRCReturn(rc, rc);
4078
4079	/*
4080	* Host FS base and GS base.
4081	*/
4082	#if HC_ARCH_BITS == 64
4083	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4084	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4085	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4086	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4087	AssertRCReturn(rc, rc);
4088
4089	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4090	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4091	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4092	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4093	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4094	#endif
4095	return VINF_SUCCESS;
4096	}
4097
4098
4099	/**
4100	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4101	* host-state area of the VMCS.
4102	*
4103	* These MSRs will be automatically restored on the host after every successful
4104	* VM-exit.
4105	*
4106	* @returns VBox status code.
4107	* @param pVCpu The cross context virtual CPU structure.
4108	*
4109	* @remarks No-long-jump zone!!!
4110	*/
4111	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4112	{
4113	AssertPtr(pVCpu);
4114
4115	/*
4116	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4117	* rather than swapping them on every VM-entry.
4118	*/
4119	hmR0VmxLazySaveHostMsrs(pVCpu);
4120
4121	/*
4122	* Host Sysenter MSRs.
4123	*/
4124	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4125	#if HC_ARCH_BITS == 32
4126	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4127	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4128	#else
4129	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4130	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4131	#endif
4132	AssertRCReturn(rc, rc);
4133
4134	/*
4135	* Host EFER MSR.
4136	*
4137	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4138	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4139	*/
4140	PVM pVM = pVCpu->CTX_SUFF(pVM);
4141	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4142	{
4143	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4144	AssertRCReturn(rc, rc);
4145	}
4146
4147	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4148	* hmR0VmxExportGuestEntryExitCtls(). */
4149
4150	return VINF_SUCCESS;
4151	}
4152
4153
4154	/**
4155	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4156	*
4157	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4158	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4159	*
4160	* @returns true if we need to load guest EFER, false otherwise.
4161	* @param pVCpu The cross context virtual CPU structure.
4162	*
4163	* @remarks Requires EFER, CR4.
4164	* @remarks No-long-jump zone!!!
4165	*/
4166	static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu)
4167	{
4168	#ifdef HMVMX_ALWAYS_SWAP_EFER
4169	RT_NOREF(pVCpu);
4170	return true;
4171	#else
4172	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4173	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4174	/* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4175	if (CPUMIsGuestInLongModeEx(pCtx))
4176	return false;
4177	#endif
4178
4179	PVM pVM = pVCpu->CTX_SUFF(pVM);
4180	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4181	uint64_t const u64GuestEfer = pCtx->msrEFER;
4182
4183	/*
4184	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4185	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4186	*/
4187	if ( CPUMIsGuestInLongModeEx(pCtx)
4188	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4189	return true;
4190
4191	/*
4192	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4193	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4194	*
4195	* See Intel spec. 4.5 "IA-32e Paging".
4196	* See Intel spec. 4.1.1 "Three Paging Modes".
4197	*
4198	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4199	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4200	*/
4201	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4202	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4203	if ( (pCtx->cr4 & X86_CR4_PAE)
4204	&& (pCtx->cr0 & X86_CR0_PG)
4205	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4206	{
4207	/* Assert that host is NX capable. */
4208	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4209	return true;
4210	}
4211
4212	return false;
4213	#endif
4214	}
4215
4216	/**
4217	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4218	* VMCS.
4219	*
4220	* This is typically required when the guest changes paging mode.
4221	*
4222	* @returns VBox status code.
4223	* @param pVCpu The cross context virtual CPU structure.
4224	* @param pVmxTransient The VMX-transient structure.
4225	*
4226	* @remarks Requires EFER.
4227	* @remarks No-long-jump zone!!!
4228	*/
4229	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4230	{
4231	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4232	{
4233	PVM pVM = pVCpu->CTX_SUFF(pVM);
4234	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4235
4236	/*
4237	* VM-entry controls.
4238	*/
4239	{
4240	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4241	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4242
4243	/*
4244	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4245	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4246	*
4247	* For nested-guests, this is a mandatory VM-entry control. It's also
4248	* required because we do not want to leak host bits to the nested-guest.
4249	*/
4250	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4251
4252	/*
4253	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4254	*
4255	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4256	* required to get the nested-guest working with hardware-assisted VMX execution.
4257	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested-hypervisor
4258	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4259	* here rather than while merging the guest VMCS controls.
4260	*/
4261	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4262	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4263	else
4264	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4265
4266	/*
4267	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4268	*
4269	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4270	* regardless of whether the nested-guest VMCS specifies it because we are free to
4271	* load whatever MSRs we require and we do not need to modify the guest visible copy
4272	* of the VM-entry MSR load area.
4273	*/
4274	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4275	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4276	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4277	else
4278	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4279
4280	/*
4281	* The following should -not- be set (since we're not in SMM mode):
4282	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4283	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4284	*/
4285
4286	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4287	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4288
4289	if ((fVal & fZap) == fVal)
4290	{ /* likely */ }
4291	else
4292	{
4293	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4294	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4295	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4296	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4297	}
4298
4299	/* Commit it to the VMCS. */
4300	if (pVmcsInfo->u32EntryCtls != fVal)
4301	{
4302	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4303	AssertRCReturn(rc, rc);
4304	pVmcsInfo->u32EntryCtls = fVal;
4305	}
4306	}
4307
4308	/*
4309	* VM-exit controls.
4310	*/
4311	{
4312	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4313	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4314
4315	/*
4316	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4317	* supported the 1-setting of this bit.
4318	*
4319	* For nested-guests, we set the "save debug controls" as the converse
4320	* "load debug controls" is mandatory for nested-guests anyway.
4321	*/
4322	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4323
4324	/*
4325	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4326	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4327	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4328	* hmR0VmxExportHostMsrs().
4329	*
4330	* For nested-guests, we always set this bit as we do not support 32-bit
4331	* hosts.
4332	*/
4333	#if HC_ARCH_BITS == 64
4334	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4335	#else
4336	Assert(!pVmxTransient->fIsNestedGuest);
4337	Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
4338	\|\| pVmcsInfo->pfnStartVM == VMXR0StartVM32);
4339	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
4340	if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
4341	{
4342	/* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
4343	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4344	}
4345	else
4346	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
4347	#endif
4348
4349	/*
4350	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4351	*
4352	* For nested-guests, we should use the "save IA32_EFER" control if we also
4353	* used the "load IA32_EFER" control while exporting VM-entry controls.
4354	*/
4355	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4356	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4357	{
4358	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4359	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4360	}
4361
4362	/*
4363	* Enable saving of the VMX-preemption timer value on VM-exit.
4364	* For nested-guests, currently not exposed/used.
4365	*/
4366	if ( pVM->hm.s.vmx.fUsePreemptTimer
4367	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4368	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4369
4370	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4371	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4372
4373	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4374	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4375	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4376
4377	if ((fVal & fZap) == fVal)
4378	{ /* likely */ }
4379	else
4380	{
4381	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4382	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4383	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4384	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4385	}
4386
4387	/* Commit it to the VMCS. */
4388	if (pVmcsInfo->u32ExitCtls != fVal)
4389	{
4390	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4391	AssertRCReturn(rc, rc);
4392	pVmcsInfo->u32ExitCtls = fVal;
4393	}
4394	}
4395
4396	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4397	}
4398	return VINF_SUCCESS;
4399	}
4400
4401
4402	/**
4403	* Sets the TPR threshold in the VMCS.
4404	*
4405	* @returns VBox status code.
4406	* @param pVCpu The cross context virtual CPU structure.
4407	* @param pVmcsInfo The VMCS info. object.
4408	* @param u32TprThreshold The TPR threshold (task-priority class only).
4409	*/
4410	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4411	{
4412	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4413	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4414	RT_NOREF2(pVCpu, pVmcsInfo);
4415	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4416	}
4417
4418
4419	/**
4420	* Exports the guest APIC TPR state into the VMCS.
4421	*
4422	* @returns VBox status code.
4423	* @param pVCpu The cross context virtual CPU structure.
4424	* @param pVmxTransient The VMX-transient structure.
4425	*
4426	* @remarks No-long-jump zone!!!
4427	*/
4428	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4429	{
4430	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4431	{
4432	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4433
4434	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4435	if (!pVmxTransient->fIsNestedGuest)
4436	{
4437	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4438	&& APICIsEnabled(pVCpu))
4439	{
4440	/*
4441	* Setup TPR shadowing.
4442	*/
4443	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4444	{
4445	bool fPendingIntr = false;
4446	uint8_t u8Tpr = 0;
4447	uint8_t u8PendingIntr = 0;
4448	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4449	AssertRCReturn(rc, rc);
4450
4451	/*
4452	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4453	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4454	* priority of the pending interrupt so we can deliver the interrupt. If there
4455	* are no interrupts pending, set threshold to 0 to not cause any
4456	* TPR-below-threshold VM-exits.
4457	*/
4458	Assert(pVmcsInfo->pbVirtApic);
4459	pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
4460	uint32_t u32TprThreshold = 0;
4461	if (fPendingIntr)
4462	{
4463	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4464	(which is the Task-Priority Class). */
4465	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4466	const uint8_t u8TprPriority = u8Tpr >> 4;
4467	if (u8PendingPriority <= u8TprPriority)
4468	u32TprThreshold = u8PendingPriority;
4469	}
4470
4471	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
4472	AssertRCReturn(rc, rc);
4473	}
4474	}
4475	}
4476	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4477	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4478	}
4479	return VINF_SUCCESS;
4480	}
4481
4482
4483	/**
4484	* Gets the guest interruptibility-state.
4485	*
4486	* @returns Guest's interruptibility-state.
4487	* @param pVCpu The cross context virtual CPU structure.
4488	* @param pVmcsInfo The VMCS info. object.
4489	*
4490	* @remarks No-long-jump zone!!!
4491	*/
4492	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
4493	{
4494	/*
4495	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4496	*/
4497	uint32_t fIntrState = 0;
4498	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4499	{
4500	/* If inhibition is active, RIP and RFLAGS should've been updated
4501	(i.e. read previously from the VMCS or from ring-3). */
4502	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4503	#ifdef VBOX_STRICT
4504	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
4505	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4506	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
4507	#endif
4508	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
4509	{
4510	if (pCtx->eflags.Bits.u1IF)
4511	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4512	else
4513	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
4514	}
4515	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4516	{
4517	/*
4518	* We can clear the inhibit force flag as even if we go back to the recompiler
4519	* without executing guest code in VT-x, the flag's condition to be cleared is
4520	* met and thus the cleared state is correct.
4521	*/
4522	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4523	}
4524	}
4525
4526	/*
4527	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
4528	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
4529	* setting this would block host-NMIs and IRET will not clear the blocking.
4530	*
4531	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
4532	*
4533	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
4534	*/
4535	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
4536	&& CPUMIsGuestNmiBlocking(pVCpu))
4537	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
4538
4539	return fIntrState;
4540	}
4541
4542
4543	/**
4544	* Exports the exception intercepts required for guest execution in the VMCS.
4545	*
4546	* @returns VBox status code.
4547	* @param pVCpu The cross context virtual CPU structure.
4548	* @param pVmxTransient The VMX-transient structure.
4549	*
4550	* @remarks No-long-jump zone!!!
4551	*/
4552	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4553	{
4554	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
4555	{
4556	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
4557	if ( !pVmxTransient->fIsNestedGuest
4558	&& pVCpu->hm.s.fGIMTrapXcptUD)
4559	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
4560	else
4561	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
4562
4563	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
4564	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
4565	}
4566	return VINF_SUCCESS;
4567	}
4568
4569
4570	/**
4571	* Exports the guest's RIP into the guest-state area in the VMCS.
4572	*
4573	* @returns VBox status code.
4574	* @param pVCpu The cross context virtual CPU structure.
4575	*
4576	* @remarks No-long-jump zone!!!
4577	*/
4578	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
4579	{
4580	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4581	{
4582	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
4583
4584	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
4585	AssertRCReturn(rc, rc);
4586
4587	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
4588	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
4589	}
4590	return VINF_SUCCESS;
4591	}
4592
4593
4594	/**
4595	* Exports the guest's RSP into the guest-state area in the VMCS.
4596	*
4597	* @returns VBox status code.
4598	* @param pVCpu The cross context virtual CPU structure.
4599	*
4600	* @remarks No-long-jump zone!!!
4601	*/
4602	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
4603	{
4604	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
4605	{
4606	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
4607
4608	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
4609	AssertRCReturn(rc, rc);
4610
4611	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
4612	}
4613	return VINF_SUCCESS;
4614	}
4615
4616
4617	/**
4618	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
4619	*
4620	* @returns VBox status code.
4621	* @param pVCpu The cross context virtual CPU structure.
4622	* @param pVmxTransient The VMX-transient structure.
4623	*
4624	* @remarks No-long-jump zone!!!
4625	*/
4626	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4627	{
4628	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
4629	{
4630	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
4631
4632	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
4633	Let us assert it as such and use 32-bit VMWRITE. */
4634	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
4635	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
4636	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
4637	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
4638
4639	/*
4640	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
4641	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
4642	* can run the real-mode guest code under Virtual 8086 mode.
4643	*/
4644	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4645	if (pVmcsInfo->RealMode.fRealOnV86Active)
4646	{
4647	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4648	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4649	Assert(!pVmxTransient->fIsNestedGuest);
4650	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
4651	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
4652	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
4653	}
4654
4655	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
4656	AssertRCReturn(rc, rc);
4657
4658	/*
4659	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
4660	*
4661	* We must avoid setting any automatic debug exceptions delivery when single-stepping
4662	* through the hypervisor debugger using EFLAGS.TF.
4663	*/
4664	if ( !pVmxTransient->fIsNestedGuest
4665	&& !pVCpu->hm.s.fSingleInstruction
4666	&& fEFlags.Bits.u1TF)
4667	{
4668	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
4669	* premature trips to ring-3 esp since IEM does not yet handle it. */
4670	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
4671	AssertRCReturn(rc, rc);
4672	}
4673	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
4674	* nested-guest VMCS. */
4675
4676	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
4677	Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
4678	}
4679	return VINF_SUCCESS;
4680	}
4681
4682
4683	/**
4684	* Exports the guest CR0 control register into the guest-state area in the VMCS.
4685	*
4686	* The guest FPU state is always pre-loaded hence we don't need to bother about
4687	* sharing FPU related CR0 bits between the guest and host.
4688	*
4689	* @returns VBox status code.
4690	* @param pVCpu The cross context virtual CPU structure.
4691	* @param pVmxTransient The VMX-transient structure.
4692	*
4693	* @remarks No-long-jump zone!!!
4694	*/
4695	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4696	{
4697	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
4698	{
4699	PVM pVM = pVCpu->CTX_SUFF(pVM);
4700	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4701
4702	/*
4703	* Figure out fixed CR0 bits in VMX operation.
4704	*/
4705	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4706	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4707	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4708	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
4709	else
4710	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
4711
4712	if (!pVmxTransient->fIsNestedGuest)
4713	{
4714	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4715	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4716	uint64_t const u64ShadowCr0 = u64GuestCr0;
4717	Assert(!RT_HI_U32(u64GuestCr0));
4718
4719	/*
4720	* Setup VT-x's view of the guest CR0.
4721	*/
4722	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
4723	if (pVM->hm.s.fNestedPaging)
4724	{
4725	if (CPUMIsGuestPagingEnabled(pVCpu))
4726	{
4727	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
4728	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
4729	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
4730	}
4731	else
4732	{
4733	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
4734	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
4735	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
4736	}
4737
4738	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
4739	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4740	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
4741	}
4742	else
4743	{
4744	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
4745	u64GuestCr0 \|= X86_CR0_WP;
4746	}
4747
4748	/*
4749	* Guest FPU bits.
4750	*
4751	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
4752	* using CR0.TS.
4753	*
4754	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
4755	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
4756	*/
4757	u64GuestCr0 \|= X86_CR0_NE;
4758
4759	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
4760	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
4761
4762	/*
4763	* Update exception intercepts.
4764	*/
4765	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
4766	if (pVmcsInfo->RealMode.fRealOnV86Active)
4767	{
4768	Assert(PDMVmmDevHeapIsEnabled(pVM));
4769	Assert(pVM->hm.s.vmx.pRealModeTSS);
4770	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
4771	}
4772	else
4773	{
4774	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
4775	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
4776	if (fInterceptMF)
4777	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
4778	}
4779
4780	/* Additional intercepts for debugging, define these yourself explicitly. */
4781	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4782	uXcptBitmap \|= 0
4783	\| RT_BIT(X86_XCPT_BP)
4784	\| RT_BIT(X86_XCPT_DE)
4785	\| RT_BIT(X86_XCPT_NM)
4786	\| RT_BIT(X86_XCPT_TS)
4787	\| RT_BIT(X86_XCPT_UD)
4788	\| RT_BIT(X86_XCPT_NP)
4789	\| RT_BIT(X86_XCPT_SS)
4790	\| RT_BIT(X86_XCPT_GP)
4791	\| RT_BIT(X86_XCPT_PF)
4792	\| RT_BIT(X86_XCPT_MF)
4793	;
4794	#elif defined(HMVMX_ALWAYS_TRAP_PF)
4795	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
4796	#endif
4797	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
4798	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
4799	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
4800
4801	/* Apply the fixed CR0 bits and enable caching. */
4802	u64GuestCr0 \|= fSetCr0;
4803	u64GuestCr0 &= fZapCr0;
4804	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4805
4806	/* Commit the CR0 and related fields to the guest VMCS. */
4807	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCr0);
4808	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4809	if (uProcCtls != pVmcsInfo->u32ProcCtls)
4810	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4811	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
4812	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
4813	AssertRCReturn(rc, rc);
4814
4815	/* Update our caches. */
4816	pVmcsInfo->u32ProcCtls = uProcCtls;
4817	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
4818
4819	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4820	}
4821	else
4822	{
4823	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
4824	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4825	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4826	uint64_t const u64ShadowCr0 = pVmcsNstGst->u64Cr0ReadShadow.u;
4827	Assert(!RT_HI_U32(u64GuestCr0));
4828	Assert(u64GuestCr0 & X86_CR0_NE);
4829
4830	/* Apply the fixed CR0 bits and enable caching. */
4831	u64GuestCr0 \|= fSetCr0;
4832	u64GuestCr0 &= fZapCr0;
4833	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4834
4835	/* Commit the CR0 and CR0 read shadow to the nested-guest VMCS. */
4836	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCr0);
4837	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4838	AssertRCReturn(rc, rc);
4839
4840	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4841	}
4842
4843	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
4844	}
4845
4846	return VINF_SUCCESS;
4847	}
4848
4849
4850	/**
4851	* Exports the guest control registers (CR3, CR4) into the guest-state area
4852	* in the VMCS.
4853	*
4854	* @returns VBox strict status code.
4855	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
4856	* without unrestricted guest access and the VMMDev is not presently
4857	* mapped (e.g. EFI32).
4858	*
4859	* @param pVCpu The cross context virtual CPU structure.
4860	* @param pVmxTransient The VMX-transient structure.
4861	*
4862	* @remarks No-long-jump zone!!!
4863	*/
4864	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4865	{
4866	int rc = VINF_SUCCESS;
4867	PVM pVM = pVCpu->CTX_SUFF(pVM);
4868
4869	/*
4870	* Guest CR2.
4871	* It's always loaded in the assembler code. Nothing to do here.
4872	*/
4873
4874	/*
4875	* Guest CR3.
4876	*/
4877	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
4878	{
4879	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
4880
4881	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
4882	if (pVM->hm.s.fNestedPaging)
4883	{
4884	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4885	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
4886
4887	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
4888	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
4889	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
4890	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
4891
4892	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
4893	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
4894	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
4895
4896	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
4897	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
4898	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
4899	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
4900	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
4901	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
4902	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
4903
4904	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
4905	AssertRCReturn(rc, rc);
4906
4907	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4908	if ( pVM->hm.s.vmx.fUnrestrictedGuest
4909	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
4910	{
4911	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
4912	if (CPUMIsGuestInPAEModeEx(pCtx))
4913	{
4914	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
4915	AssertRCReturn(rc, rc);
4916	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
4917	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
4918	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
4919	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
4920	AssertRCReturn(rc, rc);
4921	}
4922
4923	/*
4924	* The guest's view of its CR3 is unblemished with nested paging when the
4925	* guest is using paging or we have unrestricted guest execution to handle
4926	* the guest when it's not using paging.
4927	*/
4928	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
4929	GCPhysGuestCR3 = pCtx->cr3;
4930	}
4931	else
4932	{
4933	/*
4934	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
4935	* thinks it accesses physical memory directly, we use our identity-mapped
4936	* page table to map guest-linear to guest-physical addresses. EPT takes care
4937	* of translating it to host-physical addresses.
4938	*/
4939	RTGCPHYS GCPhys;
4940	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4941
4942	/* We obtain it here every time as the guest could have relocated this PCI region. */
4943	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4944	if (RT_SUCCESS(rc))
4945	{ /* likely */ }
4946	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4947	{
4948	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
4949	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4950	}
4951	else
4952	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4953
4954	GCPhysGuestCR3 = GCPhys;
4955	}
4956
4957	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
4958	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4959	AssertRCReturn(rc, rc);
4960	}
4961	else
4962	{
4963	/* Non-nested paging case, just use the hypervisor's CR3. */
4964	RTHCPHYS const HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4965
4966	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
4967	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4968	AssertRCReturn(rc, rc);
4969	}
4970
4971	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
4972	}
4973
4974	/*
4975	* Guest CR4.
4976	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4977	*/
4978	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
4979	{
4980	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4981	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4982	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
4983
4984	/*
4985	* Figure out fixed CR4 bits in VMX operation.
4986	*/
4987	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
4988	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
4989
4990	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
4991	uint64_t u64GuestCr4 = pCtx->cr4;
4992	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest ? pCtx->cr4 : pVmcsNstGst->u64Cr4ReadShadow.u;
4993	Assert(!RT_HI_U32(u64GuestCr4));
4994
4995	/*
4996	* Setup VT-x's view of the guest CR4.
4997	*
4998	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
4999	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5000	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5001	*
5002	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5003	*/
5004	if (pVmcsInfo->RealMode.fRealOnV86Active)
5005	{
5006	Assert(pVM->hm.s.vmx.pRealModeTSS);
5007	Assert(PDMVmmDevHeapIsEnabled(pVM));
5008	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5009	}
5010
5011	if (pVM->hm.s.fNestedPaging)
5012	{
5013	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5014	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5015	{
5016	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5017	u64GuestCr4 \|= X86_CR4_PSE;
5018	/* Our identity mapping is a 32-bit page directory. */
5019	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5020	}
5021	/* else use guest CR4.*/
5022	}
5023	else
5024	{
5025	Assert(!pVmxTransient->fIsNestedGuest);
5026
5027	/*
5028	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5029	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5030	*/
5031	switch (pVCpu->hm.s.enmShadowMode)
5032	{
5033	case PGMMODE_REAL: /* Real-mode. */
5034	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5035	case PGMMODE_32_BIT: /* 32-bit paging. */
5036	{
5037	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5038	break;
5039	}
5040
5041	case PGMMODE_PAE: /* PAE paging. */
5042	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5043	{
5044	u64GuestCr4 \|= X86_CR4_PAE;
5045	break;
5046	}
5047
5048	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5049	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5050	#ifdef VBOX_ENABLE_64_BITS_GUESTS
5051	break;
5052	#endif
5053	default:
5054	AssertFailed();
5055	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5056	}
5057	}
5058
5059	/* Apply the fixed CR4 bits (mainly CR4.VMXE). */
5060	u64GuestCr4 \|= fSetCr4;
5061	u64GuestCr4 &= fZapCr4;
5062
5063	/* Commit the CR4 and CR4 read shadow to the guest VMCS. */
5064	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR4, u64GuestCr4);
5065	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
5066	AssertRCReturn(rc, rc);
5067
5068	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5069	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5070
5071	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5072
5073	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5074	}
5075	return rc;
5076	}
5077
5078
5079	/**
5080	* Exports the guest debug registers into the guest-state area in the VMCS.
5081	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5082	*
5083	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5084	*
5085	* @returns VBox status code.
5086	* @param pVCpu The cross context virtual CPU structure.
5087	* @param pVmxTransient The VMX-transient structure.
5088	*
5089	* @remarks No-long-jump zone!!!
5090	*/
5091	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5092	{
5093	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5094
5095	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5096	* stepping. */
5097	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5098	if (pVmxTransient->fIsNestedGuest)
5099	{
5100	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5101	AssertRCReturn(rc, rc);
5102	return VINF_SUCCESS;
5103	}
5104
5105	#ifdef VBOX_STRICT
5106	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5107	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5108	{
5109	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5110	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5111	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5112	}
5113	#endif
5114
5115	bool fSteppingDB = false;
5116	bool fInterceptMovDRx = false;
5117	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5118	if (pVCpu->hm.s.fSingleInstruction)
5119	{
5120	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5121	PVM pVM = pVCpu->CTX_SUFF(pVM);
5122	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5123	{
5124	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5125	Assert(fSteppingDB == false);
5126	}
5127	else
5128	{
5129	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5130	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5131	pVCpu->hm.s.fClearTrapFlag = true;
5132	fSteppingDB = true;
5133	}
5134	}
5135
5136	uint32_t u32GuestDr7;
5137	if ( fSteppingDB
5138	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5139	{
5140	/*
5141	* Use the combined guest and host DRx values found in the hypervisor register set
5142	* because the hypervisor debugger has breakpoints active or someone is single stepping
5143	* on the host side without a monitor trap flag.
5144	*
5145	* Note! DBGF expects a clean DR6 state before executing guest code.
5146	*/
5147	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5148	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5149	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
5150	{
5151	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5152	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
5153	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
5154	}
5155	else
5156	#endif
5157	if (!CPUMIsHyperDebugStateActive(pVCpu))
5158	{
5159	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5160	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5161	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5162	}
5163
5164	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5165	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
5166	pVCpu->hm.s.fUsingHyperDR7 = true;
5167	fInterceptMovDRx = true;
5168	}
5169	else
5170	{
5171	/*
5172	* If the guest has enabled debug registers, we need to load them prior to
5173	* executing guest code so they'll trigger at the right time.
5174	*/
5175	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5176	{
5177	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5178	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5179	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
5180	{
5181	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5182	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
5183	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
5184	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5185	}
5186	else
5187	#endif
5188	if (!CPUMIsGuestDebugStateActive(pVCpu))
5189	{
5190	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5191	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5192	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5193	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5194	}
5195	Assert(!fInterceptMovDRx);
5196	}
5197	/*
5198	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5199	* must intercept #DB in order to maintain a correct DR6 guest value, and
5200	* because we need to intercept it to prevent nested #DBs from hanging the
5201	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5202	*/
5203	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5204	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
5205	&& !CPUMIsGuestDebugStateActive(pVCpu))
5206	#else
5207	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5208	#endif
5209	{
5210	fInterceptMovDRx = true;
5211	}
5212
5213	/* Update DR7 with the actual guest value. */
5214	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5215	pVCpu->hm.s.fUsingHyperDR7 = false;
5216	}
5217
5218	if (fInterceptMovDRx)
5219	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5220	else
5221	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5222
5223	/*
5224	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5225	* monitor-trap flag and update our cache.
5226	*/
5227	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5228	{
5229	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5230	AssertRCReturn(rc2, rc2);
5231	pVmcsInfo->u32ProcCtls = uProcCtls;
5232	}
5233
5234	/*
5235	* Update guest DR7.
5236	*/
5237	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
5238	AssertRCReturn(rc, rc);
5239
5240	/*
5241	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
5242	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
5243	*
5244	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5245	*/
5246	if (fSteppingDB)
5247	{
5248	Assert(pVCpu->hm.s.fSingleInstruction);
5249	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
5250
5251	uint32_t fIntrState = 0;
5252	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
5253	AssertRCReturn(rc, rc);
5254
5255	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5256	{
5257	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5258	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5259	AssertRCReturn(rc, rc);
5260	}
5261	}
5262
5263	return VINF_SUCCESS;
5264	}
5265
5266
5267	#ifdef VBOX_STRICT
5268	/**
5269	* Strict function to validate segment registers.
5270	*
5271	* @param pVCpu The cross context virtual CPU structure.
5272	* @param pVmcsInfo The VMCS info. object.
5273	*
5274	* @remarks Will import guest CR0 on strict builds during validation of
5275	* segments.
5276	*/
5277	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
5278	{
5279	/*
5280	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5281	*
5282	* The reason we check for attribute value 0 in this function and not just the unusable bit is
5283	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
5284	* unusable bit and doesn't change the guest-context value.
5285	*/
5286	PVM pVM = pVCpu->CTX_SUFF(pVM);
5287	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5288	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
5289	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
5290	&& ( !CPUMIsGuestInRealModeEx(pCtx)
5291	&& !CPUMIsGuestInV86ModeEx(pCtx)))
5292	{
5293	/* Protected mode checks */
5294	/* CS */
5295	Assert(pCtx->cs.Attr.n.u1Present);
5296	Assert(!(pCtx->cs.Attr.u & 0xf00));
5297	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
5298	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5299	\|\| !(pCtx->cs.Attr.n.u1Granularity));
5300	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
5301	\|\| (pCtx->cs.Attr.n.u1Granularity));
5302	/* CS cannot be loaded with NULL in protected mode. */
5303	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
5304	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5305	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
5306	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5307	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
5308	else
5309	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
5310	/* SS */
5311	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5312	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
5313	if ( !(pCtx->cr0 & X86_CR0_PE)
5314	\|\| pCtx->cs.Attr.n.u4Type == 3)
5315	{
5316	Assert(!pCtx->ss.Attr.n.u2Dpl);
5317	}
5318	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
5319	{
5320	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5321	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
5322	Assert(pCtx->ss.Attr.n.u1Present);
5323	Assert(!(pCtx->ss.Attr.u & 0xf00));
5324	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
5325	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
5326	\|\| !(pCtx->ss.Attr.n.u1Granularity));
5327	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
5328	\|\| (pCtx->ss.Attr.n.u1Granularity));
5329	}
5330	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
5331	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
5332	{
5333	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5334	Assert(pCtx->ds.Attr.n.u1Present);
5335	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
5336	Assert(!(pCtx->ds.Attr.u & 0xf00));
5337	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
5338	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
5339	\|\| !(pCtx->ds.Attr.n.u1Granularity));
5340	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
5341	\|\| (pCtx->ds.Attr.n.u1Granularity));
5342	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5343	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
5344	}
5345	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
5346	{
5347	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5348	Assert(pCtx->es.Attr.n.u1Present);
5349	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
5350	Assert(!(pCtx->es.Attr.u & 0xf00));
5351	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
5352	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
5353	\|\| !(pCtx->es.Attr.n.u1Granularity));
5354	Assert( !(pCtx->es.u32Limit & 0xfff00000)
5355	\|\| (pCtx->es.Attr.n.u1Granularity));
5356	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5357	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
5358	}
5359	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
5360	{
5361	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5362	Assert(pCtx->fs.Attr.n.u1Present);
5363	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
5364	Assert(!(pCtx->fs.Attr.u & 0xf00));
5365	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
5366	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
5367	\|\| !(pCtx->fs.Attr.n.u1Granularity));
5368	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
5369	\|\| (pCtx->fs.Attr.n.u1Granularity));
5370	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5371	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5372	}
5373	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
5374	{
5375	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5376	Assert(pCtx->gs.Attr.n.u1Present);
5377	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
5378	Assert(!(pCtx->gs.Attr.u & 0xf00));
5379	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
5380	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
5381	\|\| !(pCtx->gs.Attr.n.u1Granularity));
5382	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
5383	\|\| (pCtx->gs.Attr.n.u1Granularity));
5384	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5385	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5386	}
5387	/* 64-bit capable CPUs. */
5388	# if HC_ARCH_BITS == 64
5389	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5390	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
5391	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
5392	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
5393	# endif
5394	}
5395	else if ( CPUMIsGuestInV86ModeEx(pCtx)
5396	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
5397	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
5398	{
5399	/* Real and v86 mode checks. */
5400	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
5401	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
5402	if (pVmcsInfo->RealMode.fRealOnV86Active)
5403	{
5404	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
5405	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
5406	}
5407	else
5408	{
5409	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
5410	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
5411	}
5412
5413	/* CS */
5414	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
5415	Assert(pCtx->cs.u32Limit == 0xffff);
5416	Assert(u32CSAttr == 0xf3);
5417	/* SS */
5418	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
5419	Assert(pCtx->ss.u32Limit == 0xffff);
5420	Assert(u32SSAttr == 0xf3);
5421	/* DS */
5422	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
5423	Assert(pCtx->ds.u32Limit == 0xffff);
5424	Assert(u32DSAttr == 0xf3);
5425	/* ES */
5426	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
5427	Assert(pCtx->es.u32Limit == 0xffff);
5428	Assert(u32ESAttr == 0xf3);
5429	/* FS */
5430	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
5431	Assert(pCtx->fs.u32Limit == 0xffff);
5432	Assert(u32FSAttr == 0xf3);
5433	/* GS */
5434	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
5435	Assert(pCtx->gs.u32Limit == 0xffff);
5436	Assert(u32GSAttr == 0xf3);
5437	/* 64-bit capable CPUs. */
5438	# if HC_ARCH_BITS == 64
5439	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5440	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
5441	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
5442	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
5443	# endif
5444	}
5445	}
5446	#endif /* VBOX_STRICT */
5447
5448
5449	/**
5450	* Exports a guest segment register into the guest-state area in the VMCS.
5451	*
5452	* @returns VBox status code.
5453	* @param pVCpu The cross context virtual CPU structure.
5454	* @param pVmcsInfo The VMCS info. object.
5455	* @param iSegReg The segment register number (X86_SREG_XXX).
5456	* @param pSelReg Pointer to the segment selector.
5457	*
5458	* @remarks No-long-jump zone!!!
5459	*/
5460	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
5461	{
5462	Assert(iSegReg < X86_SREG_COUNT);
5463	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
5464	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
5465	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
5466	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
5467
5468	uint32_t u32Access = pSelReg->Attr.u;
5469	if (pVmcsInfo->RealMode.fRealOnV86Active)
5470	{
5471	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
5472	u32Access = 0xf3;
5473	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5474	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5475	RT_NOREF_PV(pVCpu);
5476	}
5477	else
5478	{
5479	/*
5480	* The way to differentiate between whether this is really a null selector or was just
5481	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
5482	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
5483	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
5484	* NULL selectors loaded in protected-mode have their attribute as 0.
5485	*/
5486	if (!u32Access)
5487	u32Access = X86DESCATTR_UNUSABLE;
5488	}
5489
5490	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
5491	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
5492	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
5493
5494	/*
5495	* Commit it to the VMCS.
5496	*/
5497	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
5498	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
5499	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
5500	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
5501	AssertRCReturn(rc, rc);
5502	return rc;
5503	}
5504
5505
5506	/**
5507	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
5508	* area in the VMCS.
5509	*
5510	* @returns VBox status code.
5511	* @param pVCpu The cross context virtual CPU structure.
5512	* @param pVmxTransient The VMX-transient structure.
5513	*
5514	* @remarks Will import guest CR0 on strict builds during validation of
5515	* segments.
5516	* @remarks No-long-jump zone!!!
5517	*/
5518	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5519	{
5520	int rc = VERR_INTERNAL_ERROR_5;
5521	PVM pVM = pVCpu->CTX_SUFF(pVM);
5522	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5523	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5524
5525	/*
5526	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
5527	*/
5528	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
5529	{
5530	#ifdef VBOX_WITH_REM
5531	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
5532	{
5533	Assert(!pVmxTransient->fIsNestedGuest);
5534	Assert(pVM->hm.s.vmx.pRealModeTSS);
5535	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
5536	if ( pVmcsInfo->fWasInRealMode
5537	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
5538	{
5539	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
5540	in real-mode (e.g. OpenBSD 4.0) */
5541	REMFlushTBs(pVM);
5542	Log4Func(("Switch to protected mode detected!\n"));
5543	pVmcsInfo->fWasInRealMode = false;
5544	}
5545	}
5546	#endif
5547	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
5548	{
5549	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
5550	if (pVmcsInfo->RealMode.fRealOnV86Active)
5551	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
5552	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
5553	AssertRCReturn(rc, rc);
5554	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
5555	}
5556
5557	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
5558	{
5559	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
5560	if (pVmcsInfo->RealMode.fRealOnV86Active)
5561	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
5562	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
5563	AssertRCReturn(rc, rc);
5564	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
5565	}
5566
5567	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
5568	{
5569	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
5570	if (pVmcsInfo->RealMode.fRealOnV86Active)
5571	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
5572	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
5573	AssertRCReturn(rc, rc);
5574	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
5575	}
5576
5577	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
5578	{
5579	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
5580	if (pVmcsInfo->RealMode.fRealOnV86Active)
5581	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
5582	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
5583	AssertRCReturn(rc, rc);
5584	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
5585	}
5586
5587	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
5588	{
5589	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
5590	if (pVmcsInfo->RealMode.fRealOnV86Active)
5591	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
5592	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
5593	AssertRCReturn(rc, rc);
5594	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
5595	}
5596
5597	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
5598	{
5599	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
5600	if (pVmcsInfo->RealMode.fRealOnV86Active)
5601	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
5602	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
5603	AssertRCReturn(rc, rc);
5604	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
5605	}
5606
5607	#ifdef VBOX_STRICT
5608	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
5609	#endif
5610	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
5611	pCtx->cs.Attr.u));
5612	}
5613
5614	/*
5615	* Guest TR.
5616	*/
5617	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
5618	{
5619	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
5620
5621	/*
5622	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
5623	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
5624	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
5625	*/
5626	uint16_t u16Sel;
5627	uint32_t u32Limit;
5628	uint64_t u64Base;
5629	uint32_t u32AccessRights;
5630	if (!pVmcsInfo->RealMode.fRealOnV86Active)
5631	{
5632	u16Sel = pCtx->tr.Sel;
5633	u32Limit = pCtx->tr.u32Limit;
5634	u64Base = pCtx->tr.u64Base;
5635	u32AccessRights = pCtx->tr.Attr.u;
5636	}
5637	else
5638	{
5639	Assert(!pVmxTransient->fIsNestedGuest);
5640	Assert(pVM->hm.s.vmx.pRealModeTSS);
5641	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
5642
5643	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
5644	RTGCPHYS GCPhys;
5645	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
5646	AssertRCReturn(rc, rc);
5647
5648	X86DESCATTR DescAttr;
5649	DescAttr.u = 0;
5650	DescAttr.n.u1Present = 1;
5651	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
5652
5653	u16Sel = 0;
5654	u32Limit = HM_VTX_TSS_SIZE;
5655	u64Base = GCPhys;
5656	u32AccessRights = DescAttr.u;
5657	}
5658
5659	/* Validate. */
5660	Assert(!(u16Sel & RT_BIT(2)));
5661	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
5662	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
5663	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
5664	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
5665	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
5666	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
5667	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
5668	Assert( (u32Limit & 0xfff) == 0xfff
5669	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
5670	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
5671	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
5672
5673	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
5674	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
5675	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
5676	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
5677	AssertRCReturn(rc, rc);
5678
5679	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
5680	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
5681	}
5682
5683	/*
5684	* Guest GDTR.
5685	*/
5686	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
5687	{
5688	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
5689
5690	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
5691	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
5692	AssertRCReturn(rc, rc);
5693
5694	/* Validate. */
5695	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5696
5697	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
5698	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
5699	}
5700
5701	/*
5702	* Guest LDTR.
5703	*/
5704	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
5705	{
5706	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
5707
5708	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
5709	uint32_t u32Access;
5710	if ( !pVmxTransient->fIsNestedGuest
5711	&& !pCtx->ldtr.Attr.u)
5712	u32Access = X86DESCATTR_UNUSABLE;
5713	else
5714	u32Access = pCtx->ldtr.Attr.u;
5715
5716	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
5717	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
5718	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
5719	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
5720	AssertRCReturn(rc, rc);
5721
5722	/* Validate. */
5723	if (!(u32Access & X86DESCATTR_UNUSABLE))
5724	{
5725	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
5726	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
5727	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
5728	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
5729	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
5730	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
5731	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
5732	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
5733	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
5734	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
5735	}
5736
5737	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
5738	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
5739	}
5740
5741	/*
5742	* Guest IDTR.
5743	*/
5744	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
5745	{
5746	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
5747
5748	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
5749	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
5750	AssertRCReturn(rc, rc);
5751
5752	/* Validate. */
5753	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5754
5755	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
5756	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
5757	}
5758
5759	return VINF_SUCCESS;
5760	}
5761
5762
5763	/**
5764	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
5765	* areas.
5766	*
5767	* These MSRs will automatically be loaded to the host CPU on every successful
5768	* VM-entry and stored from the host CPU on every successful VM-exit.
5769	*
5770	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
5771	* actual host MSR values are not- updated here for performance reasons. See
5772	* hmR0VmxExportHostMsrs().
5773	*
5774	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
5775	*
5776	* @returns VBox status code.
5777	* @param pVCpu The cross context virtual CPU structure.
5778	* @param pVmxTransient The VMX-transient structure.
5779	*
5780	* @remarks No-long-jump zone!!!
5781	*/
5782	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5783	{
5784	AssertPtr(pVCpu);
5785	AssertPtr(pVmxTransient);
5786
5787	PVM pVM = pVCpu->CTX_SUFF(pVM);
5788	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5789
5790	/*
5791	* MSRs that we use the auto-load/store MSR area in the VMCS.
5792	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
5793	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
5794	*
5795	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
5796	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
5797	* emulation, nothing to do here.
5798	*/
5799	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
5800	{
5801	if ( !pVmxTransient->fIsNestedGuest
5802	&& pVM->hm.s.fAllow64BitGuests)
5803	{
5804	#if HC_ARCH_BITS == 32
5805	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
5806	Assert(!pVmxTransient->fIsNestedGuest);
5807
5808	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
5809	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
5810	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
5811	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
5812	AssertRCReturn(rc, rc);
5813	#endif
5814	}
5815	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
5816	}
5817
5818	/*
5819	* Guest Sysenter MSRs.
5820	*/
5821	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
5822	{
5823	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
5824
5825	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
5826	{
5827	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
5828	AssertRCReturn(rc, rc);
5829	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
5830	}
5831
5832	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
5833	{
5834	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
5835	AssertRCReturn(rc, rc);
5836	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
5837	}
5838
5839	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
5840	{
5841	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
5842	AssertRCReturn(rc, rc);
5843	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
5844	}
5845	}
5846
5847	/*
5848	* Guest/host EFER MSR.
5849	*/
5850	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
5851	{
5852	/* Whether we are using the VMCS to swap the EFER MSR must have been
5853	determined earlier while exporting VM-entry/VM-exit controls. */
5854	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
5855	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
5856
5857	if (hmR0VmxShouldSwapEferMsr(pVCpu))
5858	{
5859	/*
5860	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
5861	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
5862	*/
5863	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
5864	{
5865	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
5866	AssertRCReturn(rc, rc);
5867	}
5868	else
5869	{
5870	/*
5871	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
5872	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
5873	*/
5874	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
5875	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5876	AssertRCReturn(rc, rc);
5877	}
5878	}
5879	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
5880	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
5881
5882	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
5883	}
5884
5885	/*
5886	* Other MSRs.
5887	* Speculation Control (R/W).
5888	*/
5889	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
5890	{
5891	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
5892	if (pVM->cpum.ro.GuestFeatures.fIbrs)
5893	{
5894	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
5895	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5896	AssertRCReturn(rc, rc);
5897	}
5898	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
5899	}
5900
5901	return VINF_SUCCESS;
5902	}
5903
5904
5905	/**
5906	* Selects up the appropriate function to run guest code.
5907	*
5908	* @returns VBox status code.
5909	* @param pVCpu The cross context virtual CPU structure.
5910	* @param pVmxTransient The VMX-transient structure.
5911	*
5912	* @remarks No-long-jump zone!!!
5913	*/
5914	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5915	{
5916	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5917	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5918
5919	if (CPUMIsGuestInLongModeEx(pCtx))
5920	{
5921	#ifndef VBOX_ENABLE_64_BITS_GUESTS
5922	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5923	#endif
5924	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
5925	#if HC_ARCH_BITS == 32
5926	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
5927	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
5928	{
5929	#ifdef VBOX_STRICT
5930	if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5931	{
5932	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5933	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5934	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5935	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5936	("fCtxChanged=%#RX64\n", fCtxChanged));
5937	}
5938	#endif
5939	pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
5940
5941	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
5942	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
5943	pVmcsInfo->fSwitchedTo64on32 = true;
5944	Log4Func(("Selected 64-bit switcher\n"));
5945	}
5946	#else
5947	/* 64-bit host. */
5948	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
5949	#endif
5950	}
5951	else
5952	{
5953	/* Guest is not in long mode, use the 32-bit handler. */
5954	#if HC_ARCH_BITS == 32
5955	if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
5956	&& !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
5957	&& pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5958	{
5959	# ifdef VBOX_STRICT
5960	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5961	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5962	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5963	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5964	("fCtxChanged=%#RX64\n", fCtxChanged));
5965	# endif
5966	}
5967	# ifdef VBOX_ENABLE_64_BITS_GUESTS
5968	/*
5969	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
5970	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
5971	* switcher flag now because we know the guest is in a sane state where it's safe
5972	* to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
5973	* the much faster 32-bit switcher again.
5974	*/
5975	if (!pVmcsInfo->fSwitchedTo64on32)
5976	{
5977	if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
5978	Log4Func(("Selected 32-bit switcher\n"));
5979	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5980	}
5981	else
5982	{
5983	Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
5984	if ( pVmcsInfo->RealMode.fRealOnV86Active
5985	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
5986	{
5987	pVmcsInfo->fSwitchedTo64on32 = false;
5988	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5989	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
5990	\| HM_CHANGED_VMX_ENTRY_EXIT_CTLS
5991	\| HM_CHANGED_HOST_CONTEXT);
5992	Log4Func(("Selected 32-bit switcher (safe)\n"));
5993	}
5994	}
5995	# else
5996	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5997	# endif
5998	#else
5999	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6000	#endif
6001	}
6002	Assert(pVmcsInfo->pfnStartVM);
6003	return VINF_SUCCESS;
6004	}
6005
6006
6007	/**
6008	* Wrapper for running the guest code in VT-x.
6009	*
6010	* @returns VBox status code, no informational status codes.
6011	* @param pVCpu The cross context virtual CPU structure.
6012	* @param pVmxTransient The VMX-transient structure.
6013	*
6014	* @remarks No-long-jump zone!!!
6015	*/
6016	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6017	{
6018	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6019	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6020	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6021
6022	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6023
6024	/*
6025	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6026	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6027	* callee-saved and thus the need for this XMM wrapper.
6028	*
6029	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6030	*/
6031	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6032	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6033	PVM pVM = pVCpu->CTX_SUFF(pVM);
6034	#ifdef VBOX_WITH_KERNEL_USING_XMM
6035	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6036	#else
6037	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
6038	#endif
6039	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6040	return rc;
6041	}
6042
6043
6044	/**
6045	* Reports world-switch error and dumps some useful debug info.
6046	*
6047	* @param pVCpu The cross context virtual CPU structure.
6048	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6049	* @param pVmxTransient The VMX-transient structure (only
6050	* exitReason updated).
6051	*/
6052	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6053	{
6054	Assert(pVCpu);
6055	Assert(pVmxTransient);
6056	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6057
6058	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6059	switch (rcVMRun)
6060	{
6061	case VERR_VMX_INVALID_VMXON_PTR:
6062	AssertFailed();
6063	break;
6064	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6065	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6066	{
6067	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6068	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6069	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
6070	AssertRC(rc);
6071
6072	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6073	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6074	Cannot do it here as we may have been long preempted. */
6075
6076	#ifdef VBOX_STRICT
6077	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6078	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6079	pVmxTransient->uExitReason));
6080	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6081	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6082	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6083	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6084	else
6085	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6086	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6087	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6088
6089	/* VMX control bits. */
6090	uint32_t u32Val;
6091	uint64_t u64Val;
6092	RTHCUINTREG uHCReg;
6093	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
6094	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
6095	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
6096	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
6097	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
6098	{
6099	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
6100	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
6101	}
6102	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
6103	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
6104	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
6105	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
6106	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
6107	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
6108	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
6109	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
6110	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
6111	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
6112	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
6113	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
6114	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
6115	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
6116	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
6117	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
6118	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6119	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
6120	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6121	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
6122	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
6123	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
6124	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
6125	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
6126	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
6127	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
6128	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
6129	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
6130	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
6131	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6132	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
6133	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
6134	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
6135	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6136	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6137	{
6138	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
6139	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
6140	}
6141
6142	/* Guest bits. */
6143	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
6144	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
6145	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
6146	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
6147	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
6148	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
6149	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
6150	{
6151	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
6152	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
6153	}
6154
6155	/* Host bits. */
6156	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
6157	Log4(("Host CR0 %#RHr\n", uHCReg));
6158	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
6159	Log4(("Host CR3 %#RHr\n", uHCReg));
6160	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
6161	Log4(("Host CR4 %#RHr\n", uHCReg));
6162
6163	RTGDTR HostGdtr;
6164	PCX86DESCHC pDesc;
6165	ASMGetGDTR(&HostGdtr);
6166	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
6167	Log4(("Host CS %#08x\n", u32Val));
6168	if (u32Val < HostGdtr.cbGdt)
6169	{
6170	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6171	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
6172	}
6173
6174	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
6175	Log4(("Host DS %#08x\n", u32Val));
6176	if (u32Val < HostGdtr.cbGdt)
6177	{
6178	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6179	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
6180	}
6181
6182	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
6183	Log4(("Host ES %#08x\n", u32Val));
6184	if (u32Val < HostGdtr.cbGdt)
6185	{
6186	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6187	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
6188	}
6189
6190	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
6191	Log4(("Host FS %#08x\n", u32Val));
6192	if (u32Val < HostGdtr.cbGdt)
6193	{
6194	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6195	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
6196	}
6197
6198	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
6199	Log4(("Host GS %#08x\n", u32Val));
6200	if (u32Val < HostGdtr.cbGdt)
6201	{
6202	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6203	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
6204	}
6205
6206	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
6207	Log4(("Host SS %#08x\n", u32Val));
6208	if (u32Val < HostGdtr.cbGdt)
6209	{
6210	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6211	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
6212	}
6213
6214	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
6215	Log4(("Host TR %#08x\n", u32Val));
6216	if (u32Val < HostGdtr.cbGdt)
6217	{
6218	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6219	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
6220	}
6221
6222	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
6223	Log4(("Host TR Base %#RHv\n", uHCReg));
6224	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
6225	Log4(("Host GDTR Base %#RHv\n", uHCReg));
6226	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
6227	Log4(("Host IDTR Base %#RHv\n", uHCReg));
6228	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
6229	Log4(("Host SYSENTER CS %#08x\n", u32Val));
6230	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
6231	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
6232	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
6233	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
6234	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
6235	Log4(("Host RSP %#RHv\n", uHCReg));
6236	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
6237	Log4(("Host RIP %#RHv\n", uHCReg));
6238	# if HC_ARCH_BITS == 64
6239	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6240	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6241	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6242	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6243	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6244	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6245	# endif
6246	#endif /* VBOX_STRICT */
6247	break;
6248	}
6249
6250	default:
6251	/* Impossible */
6252	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6253	break;
6254	}
6255	}
6256
6257
6258	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
6259	# ifndef VMX_USE_CACHED_VMCS_ACCESSES
6260	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
6261	# endif
6262
6263	/**
6264	* Initialize the VMCS-Read cache.
6265	*
6266	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
6267	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
6268	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
6269	* (those that have a 32-bit FULL & HIGH part).
6270	*
6271	* @param pVCpu The cross context virtual CPU structure.
6272	*/
6273	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
6274	{
6275	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
6276	do { \
6277	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
6278	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
6279	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
6280	++cReadFields; \
6281	} while (0)
6282
6283	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6284	uint32_t cReadFields = 0;
6285
6286	/*
6287	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
6288	* and serve to indicate exceptions to the rules.
6289	*/
6290
6291	/* Guest-natural selector base fields. */
6292	#if 0
6293	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
6294	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
6295	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
6296	#endif
6297	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
6298	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
6299	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
6300	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
6301	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
6302	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
6303	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
6304	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
6305	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
6306	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
6307	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
6308	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
6309	#if 0
6310	/* Unused natural width guest-state fields. */
6311	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
6312	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
6313	#endif
6314	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
6315	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
6316
6317	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
6318	these 64-bit fields (using "FULL" and "HIGH" fields). */
6319	#if 0
6320	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
6321	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
6322	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
6323	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
6324	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
6325	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
6326	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
6327	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
6328	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
6329	#endif
6330
6331	/* Natural width guest-state fields. */
6332	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
6333	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
6334
6335	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6336	{
6337	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
6338	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
6339	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
6340	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
6341	}
6342	else
6343	{
6344	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
6345	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
6346	}
6347
6348	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
6349	}
6350
6351
6352	/**
6353	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
6354	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
6355	* darwin, running 64-bit guests).
6356	*
6357	* @returns VBox status code.
6358	* @param pVCpu The cross context virtual CPU structure.
6359	* @param idxField The VMCS field encoding.
6360	* @param u64Val 16, 32 or 64-bit value.
6361	*/
6362	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6363	{
6364	int rc;
6365	switch (idxField)
6366	{
6367	/*
6368	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
6369	*/
6370	/* 64-bit Control fields. */
6371	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
6372	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
6373	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
6374	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
6375	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
6376	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
6377	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
6378	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
6379	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
6380	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
6381	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
6382	case VMX_VMCS64_CTRL_EPTP_FULL:
6383	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
6384	/* 64-bit Guest-state fields. */
6385	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
6386	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
6387	case VMX_VMCS64_GUEST_PAT_FULL:
6388	case VMX_VMCS64_GUEST_EFER_FULL:
6389	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
6390	case VMX_VMCS64_GUEST_PDPTE0_FULL:
6391	case VMX_VMCS64_GUEST_PDPTE1_FULL:
6392	case VMX_VMCS64_GUEST_PDPTE2_FULL:
6393	case VMX_VMCS64_GUEST_PDPTE3_FULL:
6394	/* 64-bit Host-state fields. */
6395	case VMX_VMCS64_HOST_PAT_FULL:
6396	case VMX_VMCS64_HOST_EFER_FULL:
6397	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
6398	{
6399	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6400	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
6401	break;
6402	}
6403
6404	/*
6405	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
6406	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
6407	*/
6408	/* Natural-width Guest-state fields. */
6409	case VMX_VMCS_GUEST_CR3:
6410	case VMX_VMCS_GUEST_ES_BASE:
6411	case VMX_VMCS_GUEST_CS_BASE:
6412	case VMX_VMCS_GUEST_SS_BASE:
6413	case VMX_VMCS_GUEST_DS_BASE:
6414	case VMX_VMCS_GUEST_FS_BASE:
6415	case VMX_VMCS_GUEST_GS_BASE:
6416	case VMX_VMCS_GUEST_LDTR_BASE:
6417	case VMX_VMCS_GUEST_TR_BASE:
6418	case VMX_VMCS_GUEST_GDTR_BASE:
6419	case VMX_VMCS_GUEST_IDTR_BASE:
6420	case VMX_VMCS_GUEST_RSP:
6421	case VMX_VMCS_GUEST_RIP:
6422	case VMX_VMCS_GUEST_SYSENTER_ESP:
6423	case VMX_VMCS_GUEST_SYSENTER_EIP:
6424	{
6425	if (!(RT_HI_U32(u64Val)))
6426	{
6427	/* If this field is 64-bit, VT-x will zero out the top bits. */
6428	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6429	}
6430	else
6431	{
6432	/* Assert that only the 32->64 switcher case should ever come here. */
6433	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
6434	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
6435	}
6436	break;
6437	}
6438
6439	default:
6440	{
6441	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
6442	rc = VERR_INVALID_PARAMETER;
6443	break;
6444	}
6445	}
6446	AssertRCReturn(rc, rc);
6447	return rc;
6448	}
6449
6450
6451	/**
6452	* Queue up a VMWRITE by using the VMCS write cache.
6453	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
6454	*
6455	* @param pVCpu The cross context virtual CPU structure.
6456	* @param idxField The VMCS field encoding.
6457	* @param u64Val 16, 32 or 64-bit value.
6458	*/
6459	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6460	{
6461	AssertPtr(pVCpu);
6462	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6463
6464	AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
6465	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
6466
6467	/* Make sure there are no duplicates. */
6468	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
6469	{
6470	if (pCache->Write.aField[i] == idxField)
6471	{
6472	pCache->Write.aFieldVal[i] = u64Val;
6473	return VINF_SUCCESS;
6474	}
6475	}
6476
6477	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
6478	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
6479	pCache->Write.cValidEntries++;
6480	return VINF_SUCCESS;
6481	}
6482	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
6483
6484
6485	/**
6486	* Sets up the usage of TSC-offsetting and updates the VMCS.
6487	*
6488	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6489	* VMX-preemption timer.
6490	*
6491	* @returns VBox status code.
6492	* @param pVCpu The cross context virtual CPU structure.
6493	* @param pVmxTransient The VMX-transient structure.
6494	*
6495	* @remarks No-long-jump zone!!!
6496	*/
6497	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6498	{
6499	bool fOffsettedTsc;
6500	bool fParavirtTsc;
6501	uint64_t uTscOffset;
6502	PVM pVM = pVCpu->CTX_SUFF(pVM);
6503	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);;
6504
6505	if (pVM->hm.s.vmx.fUsePreemptTimer)
6506	{
6507	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6508
6509	/* Make sure the returned values have sane upper and lower boundaries. */
6510	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6511	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6512	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6513	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6514
6515	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6516	* preemption timers here. We probably need to clamp the preemption timer,
6517	* after converting the timer value to the host. */
6518	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6519	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6520	AssertRC(rc);
6521	}
6522	else
6523	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6524
6525	if (fParavirtTsc)
6526	{
6527	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6528	information before every VM-entry, hence disable it for performance sake. */
6529	#if 0
6530	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
6531	AssertRC(rc);
6532	#endif
6533	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
6534	}
6535
6536	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
6537	if ( fOffsettedTsc
6538	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
6539	{
6540	if (pVmxTransient->fIsNestedGuest)
6541	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
6542	if (pVmcsInfo->u64TscOffset != uTscOffset)
6543	{
6544	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
6545	AssertRC(rc);
6546	pVmcsInfo->u64TscOffset = uTscOffset;
6547	}
6548
6549	if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
6550	{
6551	uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
6552	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6553	AssertRC(rc);
6554	pVmcsInfo->u32ProcCtls = uProcCtls;
6555	}
6556	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
6557	}
6558	else
6559	{
6560	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
6561	if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
6562	{
6563	uProcCtls \|= VMX_PROC_CTLS_RDTSC_EXIT;
6564	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6565	AssertRC(rc);
6566	pVmcsInfo->u32ProcCtls = uProcCtls;
6567	}
6568	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
6569	}
6570	}
6571
6572
6573	/**
6574	* Gets the IEM exception flags for the specified vector and IDT vectoring /
6575	* VM-exit interruption info type.
6576	*
6577	* @returns The IEM exception flags.
6578	* @param uVector The event vector.
6579	* @param uVmxEventType The VMX event type.
6580	*
6581	* @remarks This function currently only constructs flags required for
6582	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
6583	* and CR2 aspects of an exception are not included).
6584	*/
6585	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
6586	{
6587	uint32_t fIemXcptFlags;
6588	switch (uVmxEventType)
6589	{
6590	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6591	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6592	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6593	break;
6594
6595	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6596	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6597	break;
6598
6599	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6600	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
6601	break;
6602
6603	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
6604	{
6605	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6606	if (uVector == X86_XCPT_BP)
6607	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
6608	else if (uVector == X86_XCPT_OF)
6609	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
6610	else
6611	{
6612	fIemXcptFlags = 0;
6613	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
6614	}
6615	break;
6616	}
6617
6618	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6619	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6620	break;
6621
6622	default:
6623	fIemXcptFlags = 0;
6624	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
6625	break;
6626	}
6627	return fIemXcptFlags;
6628	}
6629
6630
6631	/**
6632	* Sets an event as a pending event to be injected into the guest.
6633	*
6634	* @param pVCpu The cross context virtual CPU structure.
6635	* @param u32IntInfo The VM-entry interruption-information field.
6636	* @param cbInstr The VM-entry instruction length in bytes (for software
6637	* interrupts, exceptions and privileged software
6638	* exceptions).
6639	* @param u32ErrCode The VM-entry exception error code.
6640	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
6641	* page-fault.
6642	*/
6643	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
6644	RTGCUINTPTR GCPtrFaultAddress)
6645	{
6646	Assert(!pVCpu->hm.s.Event.fPending);
6647	pVCpu->hm.s.Event.fPending = true;
6648	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
6649	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
6650	pVCpu->hm.s.Event.cbInstr = cbInstr;
6651	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
6652	}
6653
6654
6655	/**
6656	* Sets an external interrupt as pending-for-injection into the VM.
6657	*
6658	* @param pVCpu The cross context virtual CPU structure.
6659	* @param u8Interrupt The external interrupt vector.
6660	*/
6661	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
6662	{
6663	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
6664	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6665	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6666	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6667	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6668	}
6669
6670
6671	/**
6672	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
6673	*
6674	* @param pVCpu The cross context virtual CPU structure.
6675	*/
6676	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
6677	{
6678	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
6679	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
6680	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6681	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6682	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6683	}
6684
6685
6686	/**
6687	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
6688	*
6689	* @param pVCpu The cross context virtual CPU structure.
6690	*/
6691	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
6692	{
6693	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
6694	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6695	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6696	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6697	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6698	}
6699
6700
6701	/**
6702	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
6703	*
6704	* @param pVCpu The cross context virtual CPU structure.
6705	*/
6706	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
6707	{
6708	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
6709	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6710	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6711	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6712	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6713	}
6714
6715
6716	/**
6717	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
6718	*
6719	* @param pVCpu The cross context virtual CPU structure.
6720	*/
6721	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
6722	{
6723	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
6724	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6725	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6726	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6727	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6728	}
6729
6730
6731	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6732	/**
6733	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
6734	*
6735	* @param pVCpu The cross context virtual CPU structure.
6736	* @param u32ErrCode The error code for the general-protection exception.
6737	*/
6738	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
6739	{
6740	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
6741	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6742	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6743	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6744	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6745	}
6746
6747
6748	/**
6749	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
6750	*
6751	* @param pVCpu The cross context virtual CPU structure.
6752	* @param u32ErrCode The error code for the stack exception.
6753	*/
6754	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
6755	{
6756	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
6757	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6758	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6759	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6760	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6761	}
6762
6763
6764	/**
6765	* Decodes the memory operand of an instruction that caused a VM-exit.
6766	*
6767	* The VM-exit qualification field provides the displacement field for memory
6768	* operand instructions, if any.
6769	*
6770	* @returns Strict VBox status code (i.e. informational status codes too).
6771	* @retval VINF_SUCCESS if the operand was successfully decoded.
6772	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
6773	* operand.
6774	* @param pVCpu The cross context virtual CPU structure.
6775	* @param uExitInstrInfo The VM-exit instruction information field.
6776	* @param enmMemAccess The memory operand's access type (read or write).
6777	* @param GCPtrDisp The instruction displacement field, if any. For
6778	* RIP-relative addressing pass RIP + displacement here.
6779	* @param pGCPtrMem Where to store the effective destination memory address.
6780	*/
6781	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
6782	PRTGCPTR pGCPtrMem)
6783	{
6784	Assert(pGCPtrMem);
6785	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
6786	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
6787	\| CPUMCTX_EXTRN_CR0);
6788
6789	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
6790	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
6791	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
6792
6793	VMXEXITINSTRINFO ExitInstrInfo;
6794	ExitInstrInfo.u = uExitInstrInfo;
6795	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
6796	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
6797	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
6798	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
6799	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
6800	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
6801	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
6802	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
6803	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
6804
6805	/*
6806	* Validate instruction information.
6807	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
6808	*/
6809	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
6810	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
6811	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
6812	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
6813	AssertLogRelMsgReturn(fIsMemOperand,
6814	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
6815
6816	/*
6817	* Compute the complete effective address.
6818	*
6819	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
6820	* See AMD spec. 4.5.2 "Segment Registers".
6821	*/
6822	RTGCPTR GCPtrMem = GCPtrDisp;
6823	if (fBaseRegValid)
6824	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
6825	if (fIdxRegValid)
6826	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
6827
6828	RTGCPTR const GCPtrOff = GCPtrMem;
6829	if ( !fIsLongMode
6830	\|\| iSegReg >= X86_SREG_FS)
6831	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
6832	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
6833
6834	/*
6835	* Validate effective address.
6836	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
6837	*/
6838	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
6839	Assert(cbAccess > 0);
6840	if (fIsLongMode)
6841	{
6842	if (X86_IS_CANONICAL(GCPtrMem))
6843	{
6844	*pGCPtrMem = GCPtrMem;
6845	return VINF_SUCCESS;
6846	}
6847
6848	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
6849	* "Data Limit Checks in 64-bit Mode". */
6850	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
6851	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6852	return VINF_HM_PENDING_XCPT;
6853	}
6854
6855	/*
6856	* This is a watered down version of iemMemApplySegment().
6857	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
6858	* and segment CPL/DPL checks are skipped.
6859	*/
6860	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
6861	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
6862	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
6863
6864	/* Check if the segment is present and usable. */
6865	if ( pSel->Attr.n.u1Present
6866	&& !pSel->Attr.n.u1Unusable)
6867	{
6868	Assert(pSel->Attr.n.u1DescType);
6869	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
6870	{
6871	/* Check permissions for the data segment. */
6872	if ( enmMemAccess == VMXMEMACCESS_WRITE
6873	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
6874	{
6875	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6876	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
6877	return VINF_HM_PENDING_XCPT;
6878	}
6879
6880	/* Check limits if it's a normal data segment. */
6881	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
6882	{
6883	if ( GCPtrFirst32 > pSel->u32Limit
6884	\|\| GCPtrLast32 > pSel->u32Limit)
6885	{
6886	Log4Func(("Data segment limit exceeded."
6887	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6888	GCPtrLast32, pSel->u32Limit));
6889	if (iSegReg == X86_SREG_SS)
6890	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6891	else
6892	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6893	return VINF_HM_PENDING_XCPT;
6894	}
6895	}
6896	else
6897	{
6898	/* Check limits if it's an expand-down data segment.
6899	Note! The upper boundary is defined by the B bit, not the G bit! */
6900	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
6901	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
6902	{
6903	Log4Func(("Expand-down data segment limit exceeded."
6904	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6905	GCPtrLast32, pSel->u32Limit));
6906	if (iSegReg == X86_SREG_SS)
6907	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6908	else
6909	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6910	return VINF_HM_PENDING_XCPT;
6911	}
6912	}
6913	}
6914	else
6915	{
6916	/* Check permissions for the code segment. */
6917	if ( enmMemAccess == VMXMEMACCESS_WRITE
6918	\|\| ( enmMemAccess == VMXMEMACCESS_READ
6919	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
6920	{
6921	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6922	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6923	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6924	return VINF_HM_PENDING_XCPT;
6925	}
6926
6927	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6928	if ( GCPtrFirst32 > pSel->u32Limit
6929	\|\| GCPtrLast32 > pSel->u32Limit)
6930	{
6931	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6932	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6933	if (iSegReg == X86_SREG_SS)
6934	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6935	else
6936	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6937	return VINF_HM_PENDING_XCPT;
6938	}
6939	}
6940	}
6941	else
6942	{
6943	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6944	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6945	return VINF_HM_PENDING_XCPT;
6946	}
6947
6948	*pGCPtrMem = GCPtrMem;
6949	return VINF_SUCCESS;
6950	}
6951
6952
6953	/**
6954	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6955	* guest attempting to execute a VMX instruction.
6956	*
6957	* @returns Strict VBox status code (i.e. informational status codes too).
6958	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
6959	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
6960	*
6961	* @param pVCpu The cross context virtual CPU structure.
6962	* @param uExitReason The VM-exit reason.
6963	*
6964	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
6965	* @remarks No-long-jump zone!!!
6966	*/
6967	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
6968	{
6969	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
6970	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
6971
6972	if ( CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx)
6973	\|\| ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6974	&& !CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)))
6975	{
6976	Log4Func(("In real/v86-mode or long-mode outside 64-bit code segment -> #UD\n"));
6977	hmR0VmxSetPendingXcptUD(pVCpu);
6978	return VINF_HM_PENDING_XCPT;
6979	}
6980
6981	if (uExitReason == VMX_EXIT_VMXON)
6982	{
6983	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
6984
6985	/*
6986	* We check CR4.VMXE because it is required to be always set while in VMX operation
6987	* by physical CPUs and our CR4 read shadow is only consulted when executing specific
6988	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
6989	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
6990	*/
6991	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
6992	{
6993	Log4Func(("CR4.VMXE is not set -> #UD\n"));
6994	hmR0VmxSetPendingXcptUD(pVCpu);
6995	return VINF_HM_PENDING_XCPT;
6996	}
6997	}
6998	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
6999	{
7000	/*
7001	* The guest has not entered VMX operation but attempted to execute a VMX instruction
7002	* (other than VMXON), we need to raise a #UD.
7003	*/
7004	Log4Func(("Not in VMX root mode -> #UD\n"));
7005	hmR0VmxSetPendingXcptUD(pVCpu);
7006	return VINF_HM_PENDING_XCPT;
7007	}
7008
7009	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
7010	{
7011	/*
7012	* The nested-guest attempted to execute a VMX instruction, cause a VM-exit and let
7013	* the guest hypervisor deal with it.
7014	*/
7015	/** @todo NSTVMX: Trigger a VM-exit */
7016	}
7017
7018	/*
7019	* VMX instructions require CPL 0 except in VMX non-root mode where the VM-exit intercept
7020	* (above) takes preceedence over the CPL check.
7021	*/
7022	if (CPUMGetGuestCPL(pVCpu) > 0)
7023	{
7024	Log4Func(("CPL > 0 -> #GP(0)\n"));
7025	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7026	return VINF_HM_PENDING_XCPT;
7027	}
7028
7029	return VINF_SUCCESS;
7030	}
7031	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7032
7033
7034	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7035	{
7036	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7037
7038	/*
7039	* If VT-x marks the segment as unusable, most other bits remain undefined:
7040	* - For CS the L, D and G bits have meaning.
7041	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7042	* - For the remaining data segments no bits are defined.
7043	*
7044	* The present bit and the unusable bit has been observed to be set at the
7045	* same time (the selector was supposed to be invalid as we started executing
7046	* a V8086 interrupt in ring-0).
7047	*
7048	* What should be important for the rest of the VBox code, is that the P bit is
7049	* cleared. Some of the other VBox code recognizes the unusable bit, but
7050	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7051	* safe side here, we'll strip off P and other bits we don't care about. If
7052	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7053	*
7054	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7055	*/
7056	#ifdef VBOX_STRICT
7057	uint32_t const uAttr = pSelReg->Attr.u;
7058	#endif
7059
7060	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7061	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7062	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7063
7064	#ifdef VBOX_STRICT
7065	VMMRZCallRing3Disable(pVCpu);
7066	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7067	# ifdef DEBUG_bird
7068	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7069	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7070	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7071	# endif
7072	VMMRZCallRing3Enable(pVCpu);
7073	NOREF(uAttr);
7074	#endif
7075	RT_NOREF2(pVCpu, idxSel);
7076	}
7077
7078
7079	/**
7080	* Imports a guest segment register from the current VMCS into the guest-CPU
7081	* context.
7082	*
7083	* @returns VBox status code.
7084	* @param pVCpu The cross context virtual CPU structure.
7085	* @param iSegReg The segment register number (X86_SREG_XXX).
7086	*
7087	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7088	* do not log!
7089	*/
7090	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7091	{
7092	Assert(iSegReg < X86_SREG_COUNT);
7093
7094	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7095	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7096	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7097	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
7098	uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
7099	#else
7100	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7101	#endif
7102	uint64_t u64Base;
7103	uint32_t u32Sel, u32Limit, u32Attr;
7104	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7105	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7106	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7107	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
7108	if (RT_SUCCESS(rc))
7109	{
7110	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7111	pSelReg->Sel = u32Sel;
7112	pSelReg->ValidSel = u32Sel;
7113	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7114	pSelReg->u32Limit = u32Limit;
7115	pSelReg->u64Base = u64Base;
7116	pSelReg->Attr.u = u32Attr;
7117	if (u32Attr & X86DESCATTR_UNUSABLE)
7118	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7119	}
7120	return rc;
7121	}
7122
7123
7124	/**
7125	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7126	*
7127	* @returns VBox status code.
7128	* @param pVCpu The cross context virtual CPU structure.
7129	*
7130	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7131	* do not log!
7132	*/
7133	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7134	{
7135	uint64_t u64Base;
7136	uint32_t u32Sel, u32Limit, u32Attr;
7137	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7138	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7139	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7140	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7141
7142	if (RT_SUCCESS(rc))
7143	{
7144	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7145	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7146	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7147	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7148	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7149	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7150	if (u32Attr & X86DESCATTR_UNUSABLE)
7151	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7152	}
7153	return rc;
7154	}
7155
7156
7157	/**
7158	* Imports the guest TR from the current VMCS into the guest-CPU context.
7159	*
7160	* @returns VBox status code.
7161	* @param pVCpu The cross context virtual CPU structure.
7162	*
7163	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7164	* do not log!
7165	*/
7166	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7167	{
7168	uint32_t u32Sel, u32Limit, u32Attr;
7169	uint64_t u64Base;
7170	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7171	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7172	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7173	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7174	AssertRCReturn(rc, rc);
7175
7176	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7177	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7178	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7179	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7180	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7181	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7182	/* TR is the only selector that can never be unusable. */
7183	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7184	return VINF_SUCCESS;
7185	}
7186
7187
7188	/**
7189	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7190	*
7191	* @returns VBox status code.
7192	* @param pVCpu The cross context virtual CPU structure.
7193	*
7194	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7195	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7196	* instead!!!
7197	*/
7198	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7199	{
7200	uint64_t u64Val;
7201	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7202	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7203	{
7204	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7205	if (RT_SUCCESS(rc))
7206	{
7207	pCtx->rip = u64Val;
7208	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7209	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7210	}
7211	return rc;
7212	}
7213	return VINF_SUCCESS;
7214	}
7215
7216
7217	/**
7218	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7219	*
7220	* @returns VBox status code.
7221	* @param pVCpu The cross context virtual CPU structure.
7222	* @param pVmcsInfo The VMCS info. object.
7223	*
7224	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7225	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7226	* instead!!!
7227	*/
7228	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7229	{
7230	uint32_t u32Val;
7231	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7232	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7233	{
7234	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7235	if (RT_SUCCESS(rc))
7236	{
7237	pCtx->eflags.u32 = u32Val;
7238
7239	/* Restore eflags for real-on-v86-mode hack. */
7240	if (pVmcsInfo->RealMode.fRealOnV86Active)
7241	{
7242	pCtx->eflags.Bits.u1VM = 0;
7243	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7244	}
7245	}
7246	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7247	return rc;
7248	}
7249	return VINF_SUCCESS;
7250	}
7251
7252
7253	/**
7254	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7255	* context.
7256	*
7257	* @returns VBox status code.
7258	* @param pVCpu The cross context virtual CPU structure.
7259	* @param pVmcsInfo The VMCS info. object.
7260	*
7261	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7262	* do not log!
7263	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7264	* instead!!!
7265	*/
7266	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7267	{
7268	uint32_t u32Val;
7269	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7270	if (RT_SUCCESS(rc))
7271	{
7272	if (!u32Val)
7273	{
7274	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7275	{
7276	rc = hmR0VmxImportGuestRip(pVCpu);
7277	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7278	AssertRCReturn(rc, rc);
7279	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7280	}
7281
7282	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7283	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7284	}
7285	else
7286	{
7287	/*
7288	* We must import RIP here to set our EM interrupt-inhibited state.
7289	* We also import RFLAGS as our code that evaluates pending interrupts
7290	* before VM-entry requires it.
7291	*/
7292	rc = hmR0VmxImportGuestRip(pVCpu);
7293	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7294	if (RT_SUCCESS(rc))
7295	{
7296	if (u32Val & ( VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
7297	\| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7298	{
7299	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7300	}
7301	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7302	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7303
7304	if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
7305	{
7306	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7307	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
7308	}
7309	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7310	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7311	}
7312	}
7313	}
7314	return rc;
7315	}
7316
7317
7318	/**
7319	* Worker for VMXR0ImportStateOnDemand.
7320	*
7321	* @returns VBox status code.
7322	* @param pVCpu The cross context virtual CPU structure.
7323	* @param pVmcsInfo The VMCS info. object.
7324	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7325	*/
7326	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7327	{
7328	#define VMXLOCAL_BREAK_RC(a_rc) \
7329	if (RT_SUCCESS(a_rc)) \
7330	{ } \
7331	else \
7332	break
7333
7334	int rc = VINF_SUCCESS;
7335	PVM pVM = pVCpu->CTX_SUFF(pVM);
7336	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7337	uint64_t u64Val;
7338	uint32_t u32Val;
7339
7340	STAM_PROFILE_ADV_START(& pVCpu->hm.s.StatImportGuestState, x);
7341
7342	/*
7343	* We disable interrupts to make the updating of the state and in particular
7344	* the fExtrn modification atomic wrt to preemption hooks.
7345	*/
7346	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7347
7348	fWhat &= pCtx->fExtrn;
7349	if (fWhat)
7350	{
7351	do
7352	{
7353	if (fWhat & CPUMCTX_EXTRN_RIP)
7354	{
7355	rc = hmR0VmxImportGuestRip(pVCpu);
7356	VMXLOCAL_BREAK_RC(rc);
7357	}
7358
7359	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7360	{
7361	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7362	VMXLOCAL_BREAK_RC(rc);
7363	}
7364
7365	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7366	{
7367	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7368	VMXLOCAL_BREAK_RC(rc);
7369	}
7370
7371	if (fWhat & CPUMCTX_EXTRN_RSP)
7372	{
7373	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
7374	VMXLOCAL_BREAK_RC(rc);
7375	pCtx->rsp = u64Val;
7376	}
7377
7378	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7379	{
7380	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7381	if (fWhat & CPUMCTX_EXTRN_CS)
7382	{
7383	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7384	rc \|= hmR0VmxImportGuestRip(pVCpu);
7385	if (fRealOnV86Active)
7386	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7387	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7388	}
7389	if (fWhat & CPUMCTX_EXTRN_SS)
7390	{
7391	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7392	if (fRealOnV86Active)
7393	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7394	}
7395	if (fWhat & CPUMCTX_EXTRN_DS)
7396	{
7397	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7398	if (fRealOnV86Active)
7399	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7400	}
7401	if (fWhat & CPUMCTX_EXTRN_ES)
7402	{
7403	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7404	if (fRealOnV86Active)
7405	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7406	}
7407	if (fWhat & CPUMCTX_EXTRN_FS)
7408	{
7409	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7410	if (fRealOnV86Active)
7411	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7412	}
7413	if (fWhat & CPUMCTX_EXTRN_GS)
7414	{
7415	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7416	if (fRealOnV86Active)
7417	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7418	}
7419	VMXLOCAL_BREAK_RC(rc);
7420	}
7421
7422	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7423	{
7424	if (fWhat & CPUMCTX_EXTRN_LDTR)
7425	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
7426
7427	if (fWhat & CPUMCTX_EXTRN_GDTR)
7428	{
7429	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
7430	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
7431	pCtx->gdtr.pGdt = u64Val;
7432	pCtx->gdtr.cbGdt = u32Val;
7433	}
7434
7435	/* Guest IDTR. */
7436	if (fWhat & CPUMCTX_EXTRN_IDTR)
7437	{
7438	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
7439	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
7440	pCtx->idtr.pIdt = u64Val;
7441	pCtx->idtr.cbIdt = u32Val;
7442	}
7443
7444	/* Guest TR. */
7445	if (fWhat & CPUMCTX_EXTRN_TR)
7446	{
7447	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7448	don't need to import that one. */
7449	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7450	rc \|= hmR0VmxImportGuestTr(pVCpu);
7451	}
7452	VMXLOCAL_BREAK_RC(rc);
7453	}
7454
7455	if (fWhat & CPUMCTX_EXTRN_DR7)
7456	{
7457	if (!pVCpu->hm.s.fUsingHyperDR7)
7458	{
7459	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
7460	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
7461	VMXLOCAL_BREAK_RC(rc);
7462	pCtx->dr[7] = u32Val;
7463	}
7464	}
7465
7466	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7467	{
7468	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
7469	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
7470	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
7471	pCtx->SysEnter.cs = u32Val;
7472	VMXLOCAL_BREAK_RC(rc);
7473	}
7474
7475	#if HC_ARCH_BITS == 64
7476	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7477	{
7478	if ( pVM->hm.s.fAllow64BitGuests
7479	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7480	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7481	}
7482
7483	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7484	{
7485	if ( pVM->hm.s.fAllow64BitGuests
7486	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7487	{
7488	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7489	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7490	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7491	}
7492	}
7493	#endif
7494
7495	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7496	#if HC_ARCH_BITS == 32
7497	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
7498	#endif
7499	)
7500	{
7501	PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7502	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7503	Assert(cMsrs == 0 \|\| pMsr != NULL);
7504	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
7505	{
7506	switch (pMsr->u32Msr)
7507	{
7508	#if HC_ARCH_BITS == 32
7509	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
7510	case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
7511	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
7512	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
7513	#endif
7514	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
7515	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
7516	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7517
7518	default:
7519	{
7520	pVCpu->hm.s.u32HMError = pMsr->u32Msr;
7521	ASMSetFlags(fEFlags);
7522	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
7523	cMsrs));
7524	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7525	}
7526	}
7527	}
7528	}
7529
7530	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7531	{
7532	uint64_t u64Shadow;
7533	if (fWhat & CPUMCTX_EXTRN_CR0)
7534	{
7535	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7536	* remove when we drop 32-bit host w/ 64-bit host support, see
7537	* @bugref{9180#c39}. */
7538	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7539	#if HC_ARCH_BITS == 32
7540	uint32_t u32Shadow;
7541	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
7542	u64Shadow = u32Shadow;
7543	#else
7544	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
7545	#endif
7546	VMXLOCAL_BREAK_RC(rc);
7547	u64Val = u32Val;
7548	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7549	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7550	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7551	CPUMSetGuestCR0(pVCpu, u64Val);
7552	VMMRZCallRing3Enable(pVCpu);
7553	}
7554
7555	if (fWhat & CPUMCTX_EXTRN_CR4)
7556	{
7557	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7558	* remove when we drop 32-bit host w/ 64-bit host support, see
7559	* @bugref{9180#c39}. */
7560	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7561	#if HC_ARCH_BITS == 32
7562	uint32_t u32Shadow;
7563	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
7564	u64Shadow = u32Shadow;
7565	#else
7566	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
7567	#endif
7568	VMXLOCAL_BREAK_RC(rc);
7569	u64Val = u32Val;
7570	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
7571	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7572	pCtx->cr4 = u64Val;
7573	}
7574
7575	if (fWhat & CPUMCTX_EXTRN_CR3)
7576	{
7577	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7578	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7579	\|\| ( pVM->hm.s.fNestedPaging
7580	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7581	{
7582	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
7583	if (pCtx->cr3 != u64Val)
7584	{
7585	pCtx->cr3 = u64Val;
7586	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7587	}
7588
7589	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7590	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7591	if (CPUMIsGuestInPAEModeEx(pCtx))
7592	{
7593	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
7594	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
7595	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
7596	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
7597	VMXLOCAL_BREAK_RC(rc);
7598	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7599	}
7600	}
7601	}
7602	}
7603	} while (0);
7604
7605	if (RT_SUCCESS(rc))
7606	{
7607	/* Update fExtrn. */
7608	pCtx->fExtrn &= ~fWhat;
7609
7610	/* If everything has been imported, clear the HM keeper bit. */
7611	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7612	{
7613	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7614	Assert(!pCtx->fExtrn);
7615	}
7616	}
7617	}
7618	else
7619	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7620
7621	ASMSetFlags(fEFlags);
7622
7623	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7624
7625	if (RT_SUCCESS(rc))
7626	{ /* likely */ }
7627	else
7628	return rc;
7629
7630	/*
7631	* Honor any pending CR3 updates.
7632	*
7633	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
7634	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7635	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7636	*
7637	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7638	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7639	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7640	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7641	*
7642	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7643	*/
7644	if (VMMRZCallRing3IsEnabled(pVCpu))
7645	{
7646	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7647	{
7648	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7649	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7650	}
7651
7652	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7653	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7654
7655	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7656	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7657	}
7658
7659	return VINF_SUCCESS;
7660	#undef VMXLOCAL_BREAK_RC
7661	}
7662
7663
7664	/**
7665	* Saves the guest state from the VMCS into the guest-CPU context.
7666	*
7667	* @returns VBox status code.
7668	* @param pVCpu The cross context virtual CPU structure.
7669	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7670	*/
7671	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
7672	{
7673	PCVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7674	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7675	}
7676
7677
7678	/**
7679	* Check per-VM and per-VCPU force flag actions that require us to go back to
7680	* ring-3 for one reason or another.
7681	*
7682	* @returns Strict VBox status code (i.e. informational status codes too)
7683	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7684	* ring-3.
7685	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7686	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7687	* interrupts)
7688	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7689	* all EMTs to be in ring-3.
7690	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7691	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7692	* to the EM loop.
7693	*
7694	* @param pVCpu The cross context virtual CPU structure.
7695	* @param fStepping Whether we are single-stepping the guest using the
7696	* hypervisor debugger.
7697	*/
7698	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
7699	{
7700	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7701
7702	/*
7703	* Update pending interrupts into the APIC's IRR.
7704	*/
7705	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7706	APICUpdatePendingInterrupts(pVCpu);
7707
7708	/*
7709	* Anything pending? Should be more likely than not if we're doing a good job.
7710	*/
7711	PVM pVM = pVCpu->CTX_SUFF(pVM);
7712	if ( !fStepping
7713	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7714	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7715	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7716	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7717	return VINF_SUCCESS;
7718
7719	/* Pending PGM C3 sync. */
7720	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7721	{
7722	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7723	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7724	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7725	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7726	if (rcStrict2 != VINF_SUCCESS)
7727	{
7728	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7729	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7730	return rcStrict2;
7731	}
7732	}
7733
7734	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7735	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7736	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7737	{
7738	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7739	int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7740	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7741	return rc2;
7742	}
7743
7744	/* Pending VM request packets, such as hardware interrupts. */
7745	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7746	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7747	{
7748	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7749	return VINF_EM_PENDING_REQUEST;
7750	}
7751
7752	/* Pending PGM pool flushes. */
7753	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7754	{
7755	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7756	return VINF_PGM_POOL_FLUSH_PENDING;
7757	}
7758
7759	/* Pending DMA requests. */
7760	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7761	{
7762	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7763	return VINF_EM_RAW_TO_R3;
7764	}
7765
7766	return VINF_SUCCESS;
7767	}
7768
7769
7770	/**
7771	* Converts any TRPM trap into a pending HM event. This is typically used when
7772	* entering from ring-3 (not longjmp returns).
7773	*
7774	* @param pVCpu The cross context virtual CPU structure.
7775	*/
7776	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7777	{
7778	Assert(TRPMHasTrap(pVCpu));
7779	Assert(!pVCpu->hm.s.Event.fPending);
7780
7781	uint8_t uVector;
7782	TRPMEVENT enmTrpmEvent;
7783	RTGCUINT uErrCode;
7784	RTGCUINTPTR GCPtrFaultAddress;
7785	uint8_t cbInstr;
7786
7787	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7788	AssertRC(rc);
7789
7790	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7791	uint32_t u32IntInfo = uVector \| VMX_EXIT_INT_INFO_VALID;
7792	if (enmTrpmEvent == TRPM_TRAP)
7793	{
7794	/** @todo r=ramshankar: TRPM currently offers no way to determine a \#DB that was
7795	* generated using INT1 (ICEBP). */
7796	switch (uVector)
7797	{
7798	case X86_XCPT_NMI:
7799	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7800	break;
7801
7802	case X86_XCPT_BP:
7803	case X86_XCPT_OF:
7804	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7805	break;
7806
7807	case X86_XCPT_PF:
7808	case X86_XCPT_DF:
7809	case X86_XCPT_TS:
7810	case X86_XCPT_NP:
7811	case X86_XCPT_SS:
7812	case X86_XCPT_GP:
7813	case X86_XCPT_AC:
7814	u32IntInfo \|= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7815	RT_FALL_THRU();
7816	default:
7817	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7818	break;
7819	}
7820	}
7821	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7822	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7823	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7824	{
7825	switch (uVector)
7826	{
7827	case X86_XCPT_BP:
7828	case X86_XCPT_OF:
7829	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7830	break;
7831
7832	default:
7833	Assert(uVector == X86_XCPT_DB);
7834	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7835	break;
7836	}
7837	}
7838	else
7839	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7840
7841	rc = TRPMResetTrap(pVCpu);
7842	AssertRC(rc);
7843	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7844	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7845
7846	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7847	}
7848
7849
7850	/**
7851	* Converts the pending HM event into a TRPM trap.
7852	*
7853	* @param pVCpu The cross context virtual CPU structure.
7854	*/
7855	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7856	{
7857	Assert(pVCpu->hm.s.Event.fPending);
7858
7859	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7860	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7861	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
7862	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7863
7864	/* If a trap was already pending, we did something wrong! */
7865	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7866
7867	/** @todo Use HMVmxEventToTrpmEventType() later. */
7868	TRPMEVENT enmTrapType;
7869	switch (uVectorType)
7870	{
7871	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7872	enmTrapType = TRPM_HARDWARE_INT;
7873	break;
7874
7875	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7876	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7877	enmTrapType = TRPM_TRAP;
7878	break;
7879
7880	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT: /* #DB (INT1/ICEBP). */
7881	Assert(uVector == X86_XCPT_DB);
7882	enmTrapType = TRPM_SOFTWARE_INT;
7883	break;
7884
7885	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP (INT3) and #OF (INTO) */
7886	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7887	enmTrapType = TRPM_SOFTWARE_INT;
7888	break;
7889
7890	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7891	enmTrapType = TRPM_SOFTWARE_INT;
7892	break;
7893
7894	default:
7895	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7896	enmTrapType = TRPM_32BIT_HACK;
7897	break;
7898	}
7899
7900	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7901
7902	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7903	AssertRC(rc);
7904
7905	if (fErrorCodeValid)
7906	TRPMSetErrorCode(pVCpu, uErrorCode);
7907
7908	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7909	&& uVector == X86_XCPT_PF)
7910	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7911	else if (enmTrapType == TRPM_SOFTWARE_INT)
7912	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7913
7914	/* We're now done converting the pending event. */
7915	pVCpu->hm.s.Event.fPending = false;
7916	}
7917
7918
7919	/**
7920	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7921	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7922	*
7923	* @param pVCpu The cross context virtual CPU structure.
7924	* @param pVmcsInfo The VMCS info. object.
7925	*/
7926	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
7927	{
7928	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7929	{
7930	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7931	{
7932	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7933	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7934	AssertRC(rc);
7935	}
7936	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
7937	}
7938
7939
7940	/**
7941	* Clears the interrupt-window exiting control in the VMCS.
7942	*
7943	* @param pVmcsInfo The VMCS info. object.
7944	*/
7945	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7946	{
7947	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7948	{
7949	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7950	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7951	}
7952	return VINF_SUCCESS;
7953	}
7954
7955
7956	/**
7957	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7958	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7959	*
7960	* @param pVCpu The cross context virtual CPU structure.
7961	* @param pVmcsInfo The VMCS info. object.
7962	*/
7963	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
7964	{
7965	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7966	{
7967	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7968	{
7969	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7970	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7971	AssertRC(rc);
7972	Log4Func(("Setup NMI-window exiting\n"));
7973	}
7974	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7975	}
7976
7977
7978	/**
7979	* Clears the NMI-window exiting control in the VMCS.
7980	*
7981	* @param pVmcsInfo The VMCS info. object.
7982	*/
7983	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7984	{
7985	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7986	{
7987	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7988	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7989	}
7990	return VINF_SUCCESS;
7991	}
7992
7993
7994	/**
7995	* Does the necessary state syncing before returning to ring-3 for any reason
7996	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7997	*
7998	* @returns VBox status code.
7999	* @param pVCpu The cross context virtual CPU structure.
8000	* @param fImportState Whether to import the guest state from the VMCS back
8001	* to the guest-CPU context.
8002	*
8003	* @remarks No-long-jmp zone!!!
8004	*/
8005	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
8006	{
8007	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8008	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8009
8010	RTCPUID idCpu = RTMpCpuId();
8011	Log4Func(("HostCpuId=%u\n", idCpu));
8012
8013	/*
8014	* !!! IMPORTANT !!!
8015	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
8016	*/
8017
8018	/* Save the guest state if necessary. */
8019	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8020	if (fImportState)
8021	{
8022	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8023	AssertRCReturn(rc, rc);
8024	}
8025
8026	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8027	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8028	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8029
8030	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8031	#ifdef VBOX_STRICT
8032	if (CPUMIsHyperDebugStateActive(pVCpu))
8033	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8034	#endif
8035	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8036	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8037	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8038
8039	#if HC_ARCH_BITS == 64
8040	/* Restore host-state bits that VT-x only restores partially. */
8041	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8042	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8043	{
8044	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8045	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8046	}
8047	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8048	#endif
8049
8050	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8051	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8052	{
8053	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8054	if (!fImportState)
8055	{
8056	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8057	AssertRCReturn(rc, rc);
8058	}
8059	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8060	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8061	}
8062	else
8063	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8064
8065	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8066	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8067
8068	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8069	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8070	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8071	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8072	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8073	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8074	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8075	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8076	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8077
8078	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8079
8080	/** @todo This partially defeats the purpose of having preemption hooks.
8081	* The problem is, deregistering the hooks should be moved to a place that
8082	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8083	* context.
8084	*/
8085	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8086	AssertRCReturn(rc, rc);
8087
8088	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8089	NOREF(idCpu);
8090	return VINF_SUCCESS;
8091	}
8092
8093
8094	/**
8095	* Leaves the VT-x session.
8096	*
8097	* @returns VBox status code.
8098	* @param pVCpu The cross context virtual CPU structure.
8099	*
8100	* @remarks No-long-jmp zone!!!
8101	*/
8102	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8103	{
8104	HM_DISABLE_PREEMPT(pVCpu);
8105	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8106	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8107	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8108
8109	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8110	and done this from the VMXR0ThreadCtxCallback(). */
8111	if (!pVCpu->hm.s.fLeaveDone)
8112	{
8113	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8114	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8115	pVCpu->hm.s.fLeaveDone = true;
8116	}
8117	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8118
8119	/*
8120	* !!! IMPORTANT !!!
8121	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8122	*/
8123
8124	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8125	/** @todo Deregistering here means we need to VMCLEAR always
8126	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8127	* for calling VMMR0ThreadCtxHookDisable here! */
8128	VMMR0ThreadCtxHookDisable(pVCpu);
8129
8130	/* Leave HM context. This takes care of local init (term). */
8131	int rc = HMR0LeaveCpu(pVCpu);
8132
8133	HM_RESTORE_PREEMPT();
8134	return rc;
8135	}
8136
8137
8138	/**
8139	* Does the necessary state syncing before doing a longjmp to ring-3.
8140	*
8141	* @returns VBox status code.
8142	* @param pVCpu The cross context virtual CPU structure.
8143	*
8144	* @remarks No-long-jmp zone!!!
8145	*/
8146	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8147	{
8148	return hmR0VmxLeaveSession(pVCpu);
8149	}
8150
8151
8152	/**
8153	* Take necessary actions before going back to ring-3.
8154	*
8155	* An action requires us to go back to ring-3. This function does the necessary
8156	* steps before we can safely return to ring-3. This is not the same as longjmps
8157	* to ring-3, this is voluntary and prepares the guest so it may continue
8158	* executing outside HM (recompiler/IEM).
8159	*
8160	* @returns VBox status code.
8161	* @param pVCpu The cross context virtual CPU structure.
8162	* @param rcExit The reason for exiting to ring-3. Can be
8163	* VINF_VMM_UNKNOWN_RING3_CALL.
8164	*/
8165	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8166	{
8167	Assert(pVCpu);
8168	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8169
8170	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8171	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8172	{
8173	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8174	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8175	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8176	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8177	}
8178
8179	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8180	VMMRZCallRing3Disable(pVCpu);
8181	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8182
8183	/*
8184	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8185	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8186	*
8187	* This is because execution may continue from ring-3 and we would need to inject
8188	* the event from there (hence place it back in TRPM).
8189	*/
8190	if (pVCpu->hm.s.Event.fPending)
8191	{
8192	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8193	Assert(!pVCpu->hm.s.Event.fPending);
8194
8195	/* Clear the events from the VMCS. */
8196	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8197	AssertRCReturn(rc, rc);
8198	}
8199	#ifdef VBOX_STRICT
8200	else
8201	{
8202	/*
8203	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8204	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8205	* occasionally, see @bugref{9180#c42}.
8206	*/
8207	uint32_t uEntryIntInfo;
8208	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8209	AssertRC(rc);
8210	Assert(!VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8211	}
8212	#endif
8213
8214	/*
8215	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8216	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8217	* (e.g. TPR below threshold).
8218	*/
8219	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8220	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8221	AssertRCReturn(rc, rc);
8222
8223	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8224	and if we're injecting an event we should have a TRPM trap pending. */
8225	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8226	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8227	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8228	#endif
8229
8230	/* Save guest state and restore host state bits. */
8231	rc = hmR0VmxLeaveSession(pVCpu);
8232	AssertRCReturn(rc, rc);
8233	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8234
8235	/* Thread-context hooks are unregistered at this point!!! */
8236
8237	/* Sync recompiler state. */
8238	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8239	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8240	\| CPUM_CHANGED_LDTR
8241	\| CPUM_CHANGED_GDTR
8242	\| CPUM_CHANGED_IDTR
8243	\| CPUM_CHANGED_TR
8244	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8245	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8246	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8247	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8248
8249	Assert(!pVCpu->hm.s.fClearTrapFlag);
8250
8251	/* Update the exit-to-ring 3 reason. */
8252	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8253
8254	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8255	if ( rcExit != VINF_EM_RAW_INTERRUPT
8256	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8257	{
8258	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8259	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8260	}
8261
8262	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8263
8264	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
8265	VMMRZCallRing3RemoveNotification(pVCpu);
8266	VMMRZCallRing3Enable(pVCpu);
8267
8268	return rc;
8269	}
8270
8271
8272	/**
8273	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8274	* longjump to ring-3 and possibly get preempted.
8275	*
8276	* @returns VBox status code.
8277	* @param pVCpu The cross context virtual CPU structure.
8278	* @param enmOperation The operation causing the ring-3 longjump.
8279	* @param pvUser User argument, currently unused, NULL.
8280	*/
8281	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
8282	{
8283	RT_NOREF(pvUser);
8284	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8285	{
8286	/*
8287	* !!! IMPORTANT !!!
8288	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8289	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8290	*/
8291	VMMRZCallRing3RemoveNotification(pVCpu);
8292	VMMRZCallRing3Disable(pVCpu);
8293	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
8294	RTThreadPreemptDisable(&PreemptState);
8295
8296	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8297	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8298	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8299	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8300
8301	#if HC_ARCH_BITS == 64
8302	/* Restore host-state bits that VT-x only restores partially. */
8303	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8304	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8305	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8306	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8307	#endif
8308
8309	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8310	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8311	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8312
8313	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8314	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8315	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8316
8317	/* Clear the current VMCS data back to memory. */
8318	hmR0VmxClearVmcs(pVmcsInfo);
8319
8320	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8321	VMMR0ThreadCtxHookDisable(pVCpu);
8322	HMR0LeaveCpu(pVCpu);
8323	RTThreadPreemptRestore(&PreemptState);
8324	return VINF_SUCCESS;
8325	}
8326
8327	Assert(pVCpu);
8328	Assert(pvUser);
8329	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8330	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8331
8332	VMMRZCallRing3Disable(pVCpu);
8333	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8334
8335	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8336
8337	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8338	AssertRCReturn(rc, rc);
8339
8340	VMMRZCallRing3Enable(pVCpu);
8341	return VINF_SUCCESS;
8342	}
8343
8344
8345	/**
8346	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8347	* stack.
8348	*
8349	* @returns Strict VBox status code (i.e. informational status codes too).
8350	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8351	* @param pVCpu The cross context virtual CPU structure.
8352	* @param uValue The value to push to the guest stack.
8353	*/
8354	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
8355	{
8356	/*
8357	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8358	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8359	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8360	*/
8361	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8362	if (pCtx->sp == 1)
8363	return VINF_EM_RESET;
8364	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8365	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8366	AssertRC(rc);
8367	return rc;
8368	}
8369
8370
8371	/**
8372	* Injects an event into the guest upon VM-entry by updating the relevant fields
8373	* in the VM-entry area in the VMCS.
8374	*
8375	* @returns Strict VBox status code (i.e. informational status codes too).
8376	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8377	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8378	*
8379	* @param pVCpu The cross context virtual CPU structure.
8380	* @param pVmxTransient The VMX-transient structure.
8381	* @param pEvent The event being injected.
8382	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state.
8383	* This will be updated if necessary. This cannot not
8384	* be NULL.
8385	* @param fStepping Whether we're single-stepping guest execution and
8386	* should return VINF_EM_DBG_STEPPED if the event is
8387	* injected directly (registers modified by us, not by
8388	* hardware on VM-entry).
8389	*/
8390	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8391	uint32_t *pfIntrState)
8392	{
8393	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8394	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8395	Assert(pfIntrState);
8396
8397	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8398	uint32_t u32IntInfo = pEvent->u64IntInfo;
8399	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8400	uint32_t const cbInstr = pEvent->cbInstr;
8401	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8402	uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8403	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8404
8405	#ifdef VBOX_STRICT
8406	/*
8407	* Validate the error-code-valid bit for hardware exceptions.
8408	* No error codes for exceptions in real-mode.
8409	*
8410	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8411	*/
8412	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8413	&& !CPUMIsGuestInRealModeEx(pCtx))
8414	{
8415	switch (uVector)
8416	{
8417	case X86_XCPT_PF:
8418	case X86_XCPT_DF:
8419	case X86_XCPT_TS:
8420	case X86_XCPT_NP:
8421	case X86_XCPT_SS:
8422	case X86_XCPT_GP:
8423	case X86_XCPT_AC:
8424	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8425	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8426	RT_FALL_THRU();
8427	default:
8428	break;
8429	}
8430	}
8431
8432	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8433	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8434	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8435	#endif
8436
8437	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8438
8439	/*
8440	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8441	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8442	* interrupt handler in the (real-mode) guest.
8443	*
8444	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8445	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8446	*/
8447	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8448	{
8449	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8450	{
8451	/*
8452	* For CPUs with unrestricted guest execution enabled and with the guest
8453	* in real-mode, we must not set the deliver-error-code bit.
8454	*
8455	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8456	*/
8457	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8458	}
8459	else
8460	{
8461	PVM pVM = pVCpu->CTX_SUFF(pVM);
8462	Assert(PDMVmmDevHeapIsEnabled(pVM));
8463	Assert(pVM->hm.s.vmx.pRealModeTSS);
8464	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8465
8466	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8467	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8468	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8469	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8470	AssertRCReturn(rc2, rc2);
8471
8472	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8473	size_t const cbIdtEntry = sizeof(X86IDTR16);
8474	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8475	{
8476	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8477	if (uVector == X86_XCPT_DF)
8478	return VINF_EM_RESET;
8479
8480	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8481	No error codes for exceptions in real-mode. */
8482	if (uVector == X86_XCPT_GP)
8483	{
8484	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8485	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8486	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8487	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8488	HMEVENT EventXcptDf;
8489	RT_ZERO(EventXcptDf);
8490	EventXcptDf.u64IntInfo = uXcptDfInfo;
8491	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8492	}
8493
8494	/*
8495	* If we're injecting an event with no valid IDT entry, inject a #GP.
8496	* No error codes for exceptions in real-mode.
8497	*
8498	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8499	*/
8500	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8501	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8502	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8503	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8504	HMEVENT EventXcptGp;
8505	RT_ZERO(EventXcptGp);
8506	EventXcptGp.u64IntInfo = uXcptGpInfo;
8507	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8508	}
8509
8510	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8511	uint16_t uGuestIp = pCtx->ip;
8512	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8513	{
8514	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8515	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8516	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8517	}
8518	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8519	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8520
8521	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8522	X86IDTR16 IdtEntry;
8523	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8524	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8525	AssertRCReturn(rc2, rc2);
8526
8527	/* Construct the stack frame for the interrupt/exception handler. */
8528	VBOXSTRICTRC rcStrict;
8529	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8530	if (rcStrict == VINF_SUCCESS)
8531	{
8532	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8533	if (rcStrict == VINF_SUCCESS)
8534	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8535	}
8536
8537	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8538	if (rcStrict == VINF_SUCCESS)
8539	{
8540	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8541	pCtx->rip = IdtEntry.offSel;
8542	pCtx->cs.Sel = IdtEntry.uSel;
8543	pCtx->cs.ValidSel = IdtEntry.uSel;
8544	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8545	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8546	&& uVector == X86_XCPT_PF)
8547	pCtx->cr2 = GCPtrFault;
8548
8549	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8550	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8551	\| HM_CHANGED_GUEST_RSP);
8552
8553	/*
8554	* If we delivered a hardware exception (other than an NMI) and if there was
8555	* block-by-STI in effect, we should clear it.
8556	*/
8557	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8558	{
8559	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8560	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8561	Log4Func(("Clearing inhibition due to STI\n"));
8562	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8563	}
8564
8565	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8566	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8567
8568	/*
8569	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8570	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8571	*/
8572	pVCpu->hm.s.Event.fPending = false;
8573
8574	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8575	if (fStepping)
8576	rcStrict = VINF_EM_DBG_STEPPED;
8577	}
8578	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8579	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8580	return rcStrict;
8581	}
8582	}
8583
8584	/*
8585	* Validate.
8586	*/
8587	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8588	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8589
8590	/*
8591	* Inject the event into the VMCS.
8592	*/
8593	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8594	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8595	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8596	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8597	AssertRCReturn(rc, rc);
8598
8599	/*
8600	* Update guest CR2 if this is a page-fault.
8601	*/
8602	if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8603	&& uVector == X86_XCPT_PF)
8604	pCtx->cr2 = GCPtrFault;
8605
8606	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8607	return VINF_SUCCESS;
8608	}
8609
8610
8611	/**
8612	* Evaluates the event to be delivered to the guest and sets it as the pending
8613	* event.
8614	*
8615	* @returns Strict VBox status code (i.e. informational status codes too).
8616	* @param pVCpu The cross context virtual CPU structure.
8617	* @param pVmxTransient The VMX-transient structure.
8618	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8619	*/
8620	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8621	{
8622	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8623	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8624
8625	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
8626	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmcsInfo);
8627	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8628	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8629	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8630
8631	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8632	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8633	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8634	Assert(!TRPMHasTrap(pVCpu));
8635	Assert(pfIntrState);
8636
8637	*pfIntrState = fIntrState;
8638
8639	/*
8640	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
8641	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
8642	*/
8643	/** @todo SMI. SMIs take priority over NMIs. */
8644	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
8645	{
8646	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8647	if ( !pVCpu->hm.s.Event.fPending
8648	&& !fBlockNmi
8649	&& !fBlockSti
8650	&& !fBlockMovSS)
8651	{
8652	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8653	if ( pVmxTransient->fIsNestedGuest
8654	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8655	return IEMExecVmxVmexitNmi(pVCpu);
8656	#endif
8657	hmR0VmxSetPendingXcptNmi(pVCpu);
8658	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8659	Log4Func(("Pending NMI\n"));
8660	}
8661	else
8662	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8663	}
8664	/*
8665	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
8666	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
8667	*/
8668	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8669	&& !pVCpu->hm.s.fSingleInstruction)
8670	{
8671	Assert(!DBGFIsStepping(pVCpu));
8672	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8673	AssertRCReturn(rc, rc);
8674	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8675	if ( !pVCpu->hm.s.Event.fPending
8676	&& !fBlockInt
8677	&& !fBlockSti
8678	&& !fBlockMovSS)
8679	{
8680	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8681	if ( pVmxTransient->fIsNestedGuest
8682	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
8683	{
8684	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector /, true / fIntPending */);
8685	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8686	return rcStrict;
8687	}
8688	#endif
8689	uint8_t u8Interrupt;
8690	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8691	if (RT_SUCCESS(rc))
8692	{
8693	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8694	if ( pVmxTransient->fIsNestedGuest
8695	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8696	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8697	{
8698	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8699	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8700	return rcStrict;
8701	}
8702	#endif
8703	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8704	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
8705	}
8706	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8707	{
8708	if ( !pVmxTransient->fIsNestedGuest
8709	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8710	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
8711	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8712
8713	/*
8714	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8715	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8716	* need to re-set this force-flag here.
8717	*/
8718	}
8719	else
8720	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8721	}
8722	else
8723	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8724	}
8725
8726	return VINF_SUCCESS;
8727	}
8728
8729
8730	/**
8731	* Injects any pending events into the guest if the guest is in a state to
8732	* receive them.
8733	*
8734	* @returns Strict VBox status code (i.e. informational status codes too).
8735	* @param pVCpu The cross context virtual CPU structure.
8736	* @param pVmxTransient The VMX-transient structure.
8737	* @param fIntrState The VT-x guest-interruptibility state.
8738	* @param fStepping Whether we are single-stepping the guest using the
8739	* hypervisor debugger and should return
8740	* VINF_EM_DBG_STEPPED if the event was dispatched
8741	* directly.
8742	*/
8743	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
8744	{
8745	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8746	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8747
8748	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8749	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8750
8751	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8752	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8753	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8754	Assert(!TRPMHasTrap(pVCpu));
8755
8756	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8757	if (pVCpu->hm.s.Event.fPending)
8758	{
8759	/*
8760	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8761	* pending even while injecting an event and in this case, we want a VM-exit as soon as
8762	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
8763	*
8764	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8765	*/
8766	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8767	#ifdef VBOX_STRICT
8768	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8769	{
8770	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
8771	Assert(!fBlockInt);
8772	Assert(!fBlockSti);
8773	Assert(!fBlockMovSS);
8774	}
8775	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
8776	{
8777	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8778	Assert(!fBlockSti);
8779	Assert(!fBlockMovSS);
8780	Assert(!fBlockNmi);
8781	}
8782	#endif
8783	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8784	uIntType));
8785
8786	/*
8787	* Inject the event and get any changes to the guest-interruptibility state.
8788	*
8789	* The guest-interruptibility state may need to be updated if we inject the event
8790	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
8791	*/
8792	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
8793	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8794
8795	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8796	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8797	else
8798	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8799	}
8800
8801	/*
8802	* Update the guest-interruptibility state.
8803	*
8804	* This is required for the real-on-v86 software interrupt injection case above, as well as
8805	* updates to the guest state from ring-3 or IEM/REM.
8806	*/
8807	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
8808	AssertRCReturn(rc, rc);
8809
8810	/*
8811	* There's no need to clear the VM-entry interruption-information field here if we're not
8812	* injecting anything. VT-x clears the valid bit on every VM-exit.
8813	*
8814	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8815	*/
8816
8817	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8818	NOREF(fBlockMovSS); NOREF(fBlockSti);
8819	return rcStrict;
8820	}
8821
8822
8823	/**
8824	* Enters the VT-x session.
8825	*
8826	* @returns VBox status code.
8827	* @param pVCpu The cross context virtual CPU structure.
8828	*/
8829	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
8830	{
8831	AssertPtr(pVCpu);
8832	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8833	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8834
8835	LogFlowFunc(("pVCpu=%p\n", pVCpu));
8836	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8837	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8838
8839	#ifdef VBOX_STRICT
8840	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8841	RTCCUINTREG uHostCR4 = ASMGetCR4();
8842	if (!(uHostCR4 & X86_CR4_VMXE))
8843	{
8844	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8845	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8846	}
8847	#endif
8848
8849	/*
8850	* Load the appropriate VMCS as the current and active one.
8851	*/
8852	PVMXVMCSINFO pVmcsInfo;
8853	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
8854	if (!fInNestedGuestMode)
8855	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
8856	else
8857	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
8858	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
8859	if (RT_SUCCESS(rc))
8860	{
8861	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
8862	pVCpu->hm.s.fLeaveDone = false;
8863	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8864
8865	/*
8866	* Do the EMT scheduled L1D flush here if needed.
8867	*/
8868	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8869	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8870	}
8871	return rc;
8872	}
8873
8874
8875	/**
8876	* The thread-context callback (only on platforms which support it).
8877	*
8878	* @param enmEvent The thread-context event.
8879	* @param pVCpu The cross context virtual CPU structure.
8880	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8881	* @thread EMT(pVCpu)
8882	*/
8883	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8884	{
8885	NOREF(fGlobalInit);
8886
8887	switch (enmEvent)
8888	{
8889	case RTTHREADCTXEVENT_OUT:
8890	{
8891	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8892	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8893	VMCPU_ASSERT_EMT(pVCpu);
8894
8895	/* No longjmps (logger flushes, locks) in this fragile context. */
8896	VMMRZCallRing3Disable(pVCpu);
8897	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8898
8899	/* Restore host-state (FPU, debug etc.) */
8900	if (!pVCpu->hm.s.fLeaveDone)
8901	{
8902	/*
8903	* Do -not- import the guest-state here as we might already be in the middle of importing
8904	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8905	*/
8906	hmR0VmxLeave(pVCpu, false /* fImportState */);
8907	pVCpu->hm.s.fLeaveDone = true;
8908	}
8909
8910	/* Leave HM context, takes care of local init (term). */
8911	int rc = HMR0LeaveCpu(pVCpu);
8912	AssertRC(rc);
8913
8914	/* Restore longjmp state. */
8915	VMMRZCallRing3Enable(pVCpu);
8916	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8917	break;
8918	}
8919
8920	case RTTHREADCTXEVENT_IN:
8921	{
8922	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8923	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8924	VMCPU_ASSERT_EMT(pVCpu);
8925
8926	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8927	VMMRZCallRing3Disable(pVCpu);
8928	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8929
8930	/* Initialize the bare minimum state required for HM. This takes care of
8931	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8932	int rc = hmR0EnterCpu(pVCpu);
8933	AssertRC(rc);
8934	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8935	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8936
8937	/* Load the active VMCS as the current one. */
8938	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8939	rc = hmR0VmxLoadVmcs(pVmcsInfo);
8940	AssertRC(rc);
8941	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8942	pVCpu->hm.s.fLeaveDone = false;
8943
8944	/* Do the EMT scheduled L1D flush if needed. */
8945	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8946	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8947
8948	/* Restore longjmp state. */
8949	VMMRZCallRing3Enable(pVCpu);
8950	break;
8951	}
8952
8953	default:
8954	break;
8955	}
8956	}
8957
8958
8959	/**
8960	* Exports the host state into the VMCS host-state area.
8961	* Sets up the VM-exit MSR-load area.
8962	*
8963	* The CPU state will be loaded from these fields on every successful VM-exit.
8964	*
8965	* @returns VBox status code.
8966	* @param pVCpu The cross context virtual CPU structure.
8967	*
8968	* @remarks No-long-jump zone!!!
8969	*/
8970	static int hmR0VmxExportHostState(PVMCPU pVCpu)
8971	{
8972	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8973
8974	int rc = VINF_SUCCESS;
8975	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8976	{
8977	rc = hmR0VmxExportHostControlRegs();
8978	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8979
8980	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8981	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8982
8983	rc = hmR0VmxExportHostMsrs(pVCpu);
8984	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8985
8986	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8987	}
8988	return rc;
8989	}
8990
8991
8992	/**
8993	* Saves the host state in the VMCS host-state.
8994	*
8995	* @returns VBox status code.
8996	* @param pVCpu The cross context virtual CPU structure.
8997	*
8998	* @remarks No-long-jump zone!!!
8999	*/
9000	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
9001	{
9002	AssertPtr(pVCpu);
9003	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9004
9005	/*
9006	* Export the host state here while entering HM context.
9007	* When thread-context hooks are used, we might get preempted and have to re-save the host
9008	* state but most of the time we won't be, so do it here before we disable interrupts.
9009	*/
9010	return hmR0VmxExportHostState(pVCpu);
9011	}
9012
9013
9014	/**
9015	* Exports the guest state into the VMCS guest-state area.
9016	*
9017	* The will typically be done before VM-entry when the guest-CPU state and the
9018	* VMCS state may potentially be out of sync.
9019	*
9020	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9021	* VM-entry controls.
9022	* Sets up the appropriate VMX non-root function to execute guest code based on
9023	* the guest CPU mode.
9024	*
9025	* @returns VBox strict status code.
9026	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9027	* without unrestricted guest execution and the VMMDev is not presently
9028	* mapped (e.g. EFI32).
9029	*
9030	* @param pVCpu The cross context virtual CPU structure.
9031	* @param pVmxTransient The VMX-transient structure.
9032	*
9033	* @remarks No-long-jump zone!!!
9034	*/
9035	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9036	{
9037	AssertPtr(pVCpu);
9038	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9039	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9040
9041	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9042
9043	/*
9044	* Determine real-on-v86 mode.
9045	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9046	*/
9047	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9048	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9049	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9050	pVmcsInfo->RealMode. fRealOnV86Active = false;
9051	else
9052	{
9053	Assert(!pVmxTransient->fIsNestedGuest);
9054	pVmcsInfo->RealMode.fRealOnV86Active = true;
9055	}
9056
9057	/*
9058	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9059	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9060	*/
9061	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9062	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9063	* be a need to evaluate this everytime since I'm pretty sure we intercept
9064	* all guest paging mode changes. */
9065	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9066	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9067
9068	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9069	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9070
9071	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9072	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9073
9074	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9075	if (rcStrict == VINF_SUCCESS)
9076	{ /* likely */ }
9077	else
9078	{
9079	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9080	return rcStrict;
9081	}
9082
9083	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9084	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9085
9086	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9087	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9088
9089	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9090	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9091
9092	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9093	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9094
9095	rc = hmR0VmxExportGuestRip(pVCpu);
9096	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9097	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9098	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9099
9100	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9101	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9102	\| HM_CHANGED_GUEST_CR2
9103	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9104	\| HM_CHANGED_GUEST_X87
9105	\| HM_CHANGED_GUEST_SSE_AVX
9106	\| HM_CHANGED_GUEST_OTHER_XSAVE
9107	\| HM_CHANGED_GUEST_XCRx
9108	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9109	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9110	\| HM_CHANGED_GUEST_TSC_AUX
9111	\| HM_CHANGED_GUEST_OTHER_MSRS
9112	\| HM_CHANGED_GUEST_HWVIRT /* More accurate PLE handling someday? */
9113	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9114
9115	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9116	return rc;
9117	}
9118
9119
9120	/**
9121	* Exports the state shared between the host and guest into the VMCS.
9122	*
9123	* @param pVCpu The cross context virtual CPU structure.
9124	* @param pVmxTransient The VMX-transient structure.
9125	*
9126	* @remarks No-long-jump zone!!!
9127	*/
9128	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9129	{
9130	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9131	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9132
9133	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9134	{
9135	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9136	AssertRC(rc);
9137	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9138
9139	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9140	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9141	{
9142	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9143	AssertRC(rc);
9144	}
9145	}
9146
9147	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9148	{
9149	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9150	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9151	}
9152
9153	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9154	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9155	}
9156
9157
9158	/**
9159	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9160	*
9161	* @returns Strict VBox status code (i.e. informational status codes too).
9162	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9163	* without unrestricted guest execution and the VMMDev is not presently
9164	* mapped (e.g. EFI32).
9165	*
9166	* @param pVCpu The cross context virtual CPU structure.
9167	* @param pVmxTransient The VMX-transient structure.
9168	*
9169	* @remarks No-long-jump zone!!!
9170	*/
9171	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9172	{
9173	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9174	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9175	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9176
9177	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9178	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9179	#endif
9180
9181	/*
9182	* For many exits it's only RIP that changes and hence try to export it first
9183	* without going through a lot of change flag checks.
9184	*/
9185	VBOXSTRICTRC rcStrict;
9186	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9187	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9188	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9189	{
9190	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9191	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9192	{ /* likely */}
9193	else
9194	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9195	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9196	}
9197	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9198	{
9199	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9200	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9201	{ /* likely */}
9202	else
9203	{
9204	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9205	VBOXSTRICTRC_VAL(rcStrict)));
9206	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9207	return rcStrict;
9208	}
9209	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9210	}
9211	else
9212	rcStrict = VINF_SUCCESS;
9213
9214	#ifdef VBOX_STRICT
9215	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9216	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9217	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9218	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9219	("fCtxChanged=%#RX64\n", fCtxChanged));
9220	#endif
9221	return rcStrict;
9222	}
9223
9224
9225	/**
9226	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9227	* and update error record fields accordingly.
9228	*
9229	* @return VMX_IGS_* return codes.
9230	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9231	* wrong with the guest state.
9232	*
9233	* @param pVCpu The cross context virtual CPU structure.
9234	* @param pVmcsInfo The VMCS info. object.
9235	*
9236	* @remarks This function assumes our cache of the VMCS controls
9237	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9238	*/
9239	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
9240	{
9241	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9242	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9243	uError = (err); \
9244	break; \
9245	} else do { } while (0)
9246
9247	int rc;
9248	PVM pVM = pVCpu->CTX_SUFF(pVM);
9249	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9250	uint32_t uError = VMX_IGS_ERROR;
9251	uint32_t u32Val;
9252	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9253
9254	do
9255	{
9256	/*
9257	* CR0.
9258	*/
9259	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9260	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9261	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
9262	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9263	if (fUnrestrictedGuest)
9264	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
9265
9266	uint32_t u32GuestCr0;
9267	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
9268	AssertRCBreak(rc);
9269	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9270	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9271	if ( !fUnrestrictedGuest
9272	&& (u32GuestCr0 & X86_CR0_PG)
9273	&& !(u32GuestCr0 & X86_CR0_PE))
9274	{
9275	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9276	}
9277
9278	/*
9279	* CR4.
9280	*/
9281	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9282	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9283
9284	uint32_t u32GuestCr4;
9285	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
9286	AssertRCBreak(rc);
9287	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9288	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9289
9290	/*
9291	* IA32_DEBUGCTL MSR.
9292	*/
9293	uint64_t u64Val;
9294	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9295	AssertRCBreak(rc);
9296	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9297	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9298	{
9299	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9300	}
9301	uint64_t u64DebugCtlMsr = u64Val;
9302
9303	#ifdef VBOX_STRICT
9304	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9305	AssertRCBreak(rc);
9306	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9307	#endif
9308	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9309
9310	/*
9311	* RIP and RFLAGS.
9312	*/
9313	uint32_t u32Eflags;
9314	#if HC_ARCH_BITS == 64
9315	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9316	AssertRCBreak(rc);
9317	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9318	if ( !fLongModeGuest
9319	\|\| !pCtx->cs.Attr.n.u1Long)
9320	{
9321	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9322	}
9323	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9324	* must be identical if the "IA-32e mode guest" VM-entry
9325	* control is 1 and CS.L is 1. No check applies if the
9326	* CPU supports 64 linear-address bits. */
9327
9328	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9329	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9330	AssertRCBreak(rc);
9331	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9332	VMX_IGS_RFLAGS_RESERVED);
9333	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9334	u32Eflags = u64Val;
9335	#else
9336	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9337	AssertRCBreak(rc);
9338	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9339	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9340	#endif
9341
9342	if ( fLongModeGuest
9343	\|\| ( fUnrestrictedGuest
9344	&& !(u32GuestCr0 & X86_CR0_PE)))
9345	{
9346	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9347	}
9348
9349	uint32_t u32EntryInfo;
9350	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9351	AssertRCBreak(rc);
9352	if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9353	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9354	{
9355	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9356	}
9357
9358	/*
9359	* 64-bit checks.
9360	*/
9361	#if HC_ARCH_BITS == 64
9362	if (fLongModeGuest)
9363	{
9364	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9365	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9366	}
9367
9368	if ( !fLongModeGuest
9369	&& (u32GuestCr4 & X86_CR4_PCIDE))
9370	{
9371	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9372	}
9373
9374	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9375	* 51:32 beyond the processor's physical-address width are 0. */
9376
9377	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9378	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9379	{
9380	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9381	}
9382
9383	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9384	AssertRCBreak(rc);
9385	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9386
9387	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9388	AssertRCBreak(rc);
9389	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9390	#endif
9391
9392	/*
9393	* PERF_GLOBAL MSR.
9394	*/
9395	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9396	{
9397	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9398	AssertRCBreak(rc);
9399	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9400	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9401	}
9402
9403	/*
9404	* PAT MSR.
9405	*/
9406	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9407	{
9408	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9409	AssertRCBreak(rc);
9410	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9411	for (unsigned i = 0; i < 8; i++)
9412	{
9413	uint8_t u8Val = (u64Val & 0xff);
9414	if ( u8Val != 0 /* UC */
9415	&& u8Val != 1 /* WC */
9416	&& u8Val != 4 /* WT */
9417	&& u8Val != 5 /* WP */
9418	&& u8Val != 6 /* WB */
9419	&& u8Val != 7 /* UC- */)
9420	{
9421	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9422	}
9423	u64Val >>= 8;
9424	}
9425	}
9426
9427	/*
9428	* EFER MSR.
9429	*/
9430	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9431	{
9432	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9433	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9434	AssertRCBreak(rc);
9435	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9436	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9437	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9438	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9439	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9440	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9441	* iemVmxVmentryCheckGuestState(). */
9442	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9443	\|\| !(u32GuestCr0 & X86_CR0_PG)
9444	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9445	VMX_IGS_EFER_LMA_LME_MISMATCH);
9446	}
9447
9448	/*
9449	* Segment registers.
9450	*/
9451	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9452	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9453	if (!(u32Eflags & X86_EFL_VM))
9454	{
9455	/* CS */
9456	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9457	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9458	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9459	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9460	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9461	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9462	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9463	/* CS cannot be loaded with NULL in protected mode. */
9464	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9465	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9466	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9467	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9468	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9469	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9470	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9471	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9472	else
9473	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9474
9475	/* SS */
9476	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9477	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9478	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9479	if ( !(pCtx->cr0 & X86_CR0_PE)
9480	\|\| pCtx->cs.Attr.n.u4Type == 3)
9481	{
9482	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9483	}
9484	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9485	{
9486	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9487	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9488	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9489	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9490	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9491	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9492	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9493	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9494	}
9495
9496	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9497	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9498	{
9499	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9500	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9501	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9502	\|\| pCtx->ds.Attr.n.u4Type > 11
9503	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9504	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9505	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9506	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9507	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9508	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9509	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9510	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9511	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9512	}
9513	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9514	{
9515	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9516	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9517	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9518	\|\| pCtx->es.Attr.n.u4Type > 11
9519	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9520	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9521	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9522	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9523	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9524	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9525	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9526	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9527	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9528	}
9529	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9530	{
9531	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9532	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9533	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9534	\|\| pCtx->fs.Attr.n.u4Type > 11
9535	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9536	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9537	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9538	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9539	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9540	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9541	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9542	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9543	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9544	}
9545	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9546	{
9547	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9548	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9549	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9550	\|\| pCtx->gs.Attr.n.u4Type > 11
9551	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9552	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9553	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9554	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9555	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9556	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9557	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9558	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9559	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9560	}
9561	/* 64-bit capable CPUs. */
9562	#if HC_ARCH_BITS == 64
9563	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9564	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9565	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9566	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9567	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9568	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9569	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9570	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9571	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9572	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9573	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9574	#endif
9575	}
9576	else
9577	{
9578	/* V86 mode checks. */
9579	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9580	if (pVmcsInfo->RealMode.fRealOnV86Active)
9581	{
9582	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9583	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9584	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9585	}
9586	else
9587	{
9588	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9589	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9590	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9591	}
9592
9593	/* CS */
9594	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9595	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9596	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9597	/* SS */
9598	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9599	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9600	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9601	/* DS */
9602	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9603	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9604	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9605	/* ES */
9606	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9607	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9608	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9609	/* FS */
9610	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9611	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9612	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9613	/* GS */
9614	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9615	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9616	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9617	/* 64-bit capable CPUs. */
9618	#if HC_ARCH_BITS == 64
9619	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9620	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9621	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9622	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9623	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9624	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9625	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9626	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9627	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9628	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9629	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9630	#endif
9631	}
9632
9633	/*
9634	* TR.
9635	*/
9636	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9637	/* 64-bit capable CPUs. */
9638	#if HC_ARCH_BITS == 64
9639	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9640	#endif
9641	if (fLongModeGuest)
9642	{
9643	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9644	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9645	}
9646	else
9647	{
9648	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9649	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9650	VMX_IGS_TR_ATTR_TYPE_INVALID);
9651	}
9652	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9653	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9654	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9655	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9656	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9657	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9658	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9659	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9660
9661	/*
9662	* GDTR and IDTR.
9663	*/
9664	#if HC_ARCH_BITS == 64
9665	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9666	AssertRCBreak(rc);
9667	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9668
9669	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9670	AssertRCBreak(rc);
9671	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9672	#endif
9673
9674	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9675	AssertRCBreak(rc);
9676	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9677
9678	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9679	AssertRCBreak(rc);
9680	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9681
9682	/*
9683	* Guest Non-Register State.
9684	*/
9685	/* Activity State. */
9686	uint32_t u32ActivityState;
9687	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9688	AssertRCBreak(rc);
9689	HMVMX_CHECK_BREAK( !u32ActivityState
9690	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9691	VMX_IGS_ACTIVITY_STATE_INVALID);
9692	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9693	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9694	uint32_t u32IntrState;
9695	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9696	AssertRCBreak(rc);
9697	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9698	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9699	{
9700	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9701	}
9702
9703	/** @todo Activity state and injecting interrupts. Left as a todo since we
9704	* currently don't use activity states but ACTIVE. */
9705
9706	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9707	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9708
9709	/* Guest interruptibility-state. */
9710	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9711	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9712	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9713	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9714	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9715	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9716	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9717	if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
9718	{
9719	if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9720	{
9721	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9722	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9723	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9724	}
9725	else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9726	{
9727	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9728	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9729	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9730	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9731	}
9732	}
9733	/** @todo Assumes the processor is not in SMM. */
9734	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9735	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9736	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9737	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9738	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9739	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9740	&& VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9741	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9742	{
9743	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
9744	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9745	}
9746
9747	/* Pending debug exceptions. */
9748	#if HC_ARCH_BITS == 64
9749	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
9750	AssertRCBreak(rc);
9751	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9752	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9753	u32Val = u64Val; /* For pending debug exceptions checks below. */
9754	#else
9755	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
9756	AssertRCBreak(rc);
9757	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9758	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9759	#endif
9760
9761	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9762	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
9763	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9764	{
9765	if ( (u32Eflags & X86_EFL_TF)
9766	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9767	{
9768	/* Bit 14 is PendingDebug.BS. */
9769	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9770	}
9771	if ( !(u32Eflags & X86_EFL_TF)
9772	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9773	{
9774	/* Bit 14 is PendingDebug.BS. */
9775	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9776	}
9777	}
9778
9779	/* VMCS link pointer. */
9780	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9781	AssertRCBreak(rc);
9782	if (u64Val != UINT64_C(0xffffffffffffffff))
9783	{
9784	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9785	/** @todo Bits beyond the processor's physical-address width MBZ. */
9786	/** @todo 32-bit located in memory referenced by value of this field (as a
9787	* physical address) must contain the processor's VMCS revision ID. */
9788	/** @todo SMM checks. */
9789	}
9790
9791	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9792	* not using nested paging? */
9793	if ( pVM->hm.s.fNestedPaging
9794	&& !fLongModeGuest
9795	&& CPUMIsGuestInPAEModeEx(pCtx))
9796	{
9797	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9798	AssertRCBreak(rc);
9799	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9800
9801	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9802	AssertRCBreak(rc);
9803	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9804
9805	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9806	AssertRCBreak(rc);
9807	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9808
9809	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9810	AssertRCBreak(rc);
9811	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9812	}
9813
9814	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9815	if (uError == VMX_IGS_ERROR)
9816	uError = VMX_IGS_REASON_NOT_FOUND;
9817	} while (0);
9818
9819	pVCpu->hm.s.u32HMError = uError;
9820	return uError;
9821
9822	#undef HMVMX_ERROR_BREAK
9823	#undef HMVMX_CHECK_BREAK
9824	}
9825
9826
9827	/**
9828	* Setup the APIC-access page for virtualizing APIC access.
9829	*
9830	* This can cause a longjumps to R3 due to the acquisition of the PGM lock, hence
9831	* this not done as part of exporting guest state, see @bugref{8721}.
9832	*
9833	* @returns VBox status code.
9834	* @param pVCpu The cross context virtual CPU structure.
9835	*/
9836	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
9837	{
9838	PVM pVM = pVCpu->CTX_SUFF(pVM);
9839	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
9840
9841	Assert(PDMHasApic(pVM));
9842	Assert(u64MsrApicBase);
9843
9844	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
9845	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
9846
9847	/* Unalias any existing mapping. */
9848	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
9849	AssertRCReturn(rc, rc);
9850
9851	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
9852	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
9853	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
9854	AssertRCReturn(rc, rc);
9855
9856	/* Update the per-VCPU cache of the APIC base MSR. */
9857	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
9858	return VINF_SUCCESS;
9859	}
9860
9861
9862	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9863	/**
9864	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
9865	* nested-guest using hardware-assisted VMX.
9866	*
9867	* @param pVCpu The cross context virtual CPU structure.
9868	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
9869	* @param pVmcsInfoGst The guest VMCS info. object.
9870	*/
9871	static void hmR0VmxMergeMsrBitmapNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
9872	{
9873	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
9874	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
9875	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
9876	Assert(pu64MsrBitmapNstGst);
9877	Assert(pu64MsrBitmapGst);
9878	Assert(pu64MsrBitmap);
9879
9880	/*
9881	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
9882	* MSR that is intercepted by the guest is also intercepted while executing the
9883	* nested-guest using hardware-assisted VMX.
9884	*/
9885	uint32_t const cbFrag = sizeof(uint64_t);
9886	uint32_t const cFrags = X86_PAGE_4K_SIZE / cbFrag;
9887	for (uint32_t i = 0; i <= cFrags; i++)
9888	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
9889	}
9890
9891
9892	/**
9893	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
9894	* hardware-assisted VMX execution of the nested-guest.
9895	*
9896	* For a guest, we don't modify these controls once we set up the VMCS.
9897	*
9898	* For nested-guests since the guest hypervisor provides these controls on every
9899	* nested-guest VM-entry and could potentially change them everytime we need to
9900	* merge them before every nested-guest VM-entry.
9901	*
9902	* @returns VBox status code.
9903	* @param pVCpu The cross context virtual CPU structure.
9904	*/
9905	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
9906	{
9907	PVM pVM = pVCpu->CTX_SUFF(pVM);
9908	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
9909	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
9910	Assert(pVmcsNstGst);
9911
9912	/*
9913	* Merge the controls with the requirements of the guest VMCS.
9914	*
9915	* We do not need to validate the nested-guest VMX features specified in the
9916	* nested-guest VMCS with the features supported by the physical CPU as it's
9917	* already done by the VMLAUNCH/VMRESUME instruction emulation.
9918	*
9919	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the
9920	* guest are derived from the VMX features supported by the physical CPU.
9921	*/
9922
9923	/* Pin-based VM-execution controls. */
9924	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
9925
9926	/* Processor-based VM-execution controls. */
9927	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
9928	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
9929	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
9930	\| VMX_PROC_CTLS_USE_TPR_SHADOW
9931	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
9932
9933	/* Secondary processor-based VM-execution controls. */
9934	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
9935	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
9936	\| VMX_PROC_CTLS2_INVPCID
9937	\| VMX_PROC_CTLS2_RDTSCP
9938	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
9939	\| VMX_PROC_CTLS2_APIC_REG_VIRT
9940	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
9941	\| VMX_PROC_CTLS2_VMFUNC));
9942
9943	/*
9944	* VM-entry controls:
9945	* These controls contains state that depends on the nested-guest state (primarily
9946	* EFER MSR) and is thus not constant through VMLAUNCH/VMRESUME and the nested-guest
9947	* VM-exit. Although the nested-hypervisor cannot change it, we need to in order to
9948	* properly continue executing the nested-guest if the EFER MSR changes but does not
9949	* cause a nested-guest VM-exits.
9950	*
9951	* VM-exit controls:
9952	* These controls specify the host state on return. We cannot use the controls from
9953	* the nested-hypervisor state as is as it would contain the guest state rather than
9954	* the host state. Since the host state is subject to change (e.g. preemption, trips
9955	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
9956	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
9957	*
9958	* VM-entry MSR-load:
9959	* The guest MSRs from the VM-entry MSR-load area are already loaded into the
9960	* guest-CPU context by the VMLAUNCH/VMRESUME instruction emulation.
9961	*
9962	* VM-exit MSR-store:
9963	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU
9964	* context back into the VM-exit MSR-store area.
9965	*
9966	* VM-exit MSR-load areas:
9967	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence,
9968	* we can entirely ignore what the nested-hypervisor wants to load here.
9969	*/
9970
9971	/*
9972	* Exception bitmap.
9973	*
9974	* We could remove #UD from the guest bitmap and merge it with the nested-guest
9975	* bitmap here (and avoid doing anything while exporting nested-guest state), but to
9976	* keep the code more flexible if intercepting exceptions become more dynamic in
9977	* the future we do it as part of exporting the nested-guest state.
9978	*/
9979	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
9980
9981	/*
9982	* CR0/CR4 guest/host mask.
9983	*
9984	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest
9985	* must cause VM-exits, so we need to merge them here.
9986	*/
9987	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
9988	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
9989
9990	/*
9991	* Page-fault error-code mask and match.
9992	*
9993	* Although we require unrestricted guest execution (and thereby nested-paging) for
9994	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
9995	* normally intercept #PFs, it might intercept them for debugging purposes.
9996	*
9997	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF
9998	* filters. If the outer guest is intercepting #PFs we must intercept all #PFs.
9999	*/
10000	uint32_t u32XcptPFMask;
10001	uint32_t u32XcptPFMatch;
10002	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10003	{
10004	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10005	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10006	}
10007	else
10008	{
10009	u32XcptPFMask = 0;
10010	u32XcptPFMatch = 0;
10011	}
10012
10013	/*
10014	* Pause-Loop exiting.
10015	*/
10016	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10017	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10018
10019	/*
10020	* I/O Bitmap.
10021	*
10022	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we
10023	* always intercept all I/O port accesses.
10024	*/
10025	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10026
10027	/*
10028	* APIC-access page.
10029	*
10030	* The APIC-access page address has already been initialized while setting up the
10031	* nested-guest VMCS. In theory, even if the guest-physical address is invalid, it
10032	* should not be on any consequence to the host or to the guest for that matter, but
10033	* we only accept valid addresses verified by the VMLAUNCH/VMRESUME instruction
10034	* emulation to keep it simple.
10035	*/
10036
10037	/*
10038	* Virtual-APIC page and TPR threshold.
10039	*
10040	* We shall use the host-physical address of the virtual-APIC page in guest memory directly.
10041	* For this reason, we can access the virtual-APIC page of the nested-guest only using
10042	* PGM physical handlers as we must not assume a kernel virtual-address mapping exists and
10043	* requesting PGM for a mapping could be expensive/resource intensive (PGM mapping cache).
10044	*/
10045	RTHCPHYS HCPhysVirtApic = NIL_RTHCPHYS;
10046	uint32_t const u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10047	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10048	{
10049	int rc = PGMPhysGCPhys2HCPhys(pVM, pVmcsNstGst->u64AddrVirtApic.u, &HCPhysVirtApic);
10050
10051	/*
10052	* If the guest hypervisor has loaded crap into the virtual-APIC page field
10053	* we would fail to obtain a valid host-physical address for its guest-physical
10054	* address.
10055	*
10056	* We currently do not support this scenario. Maybe in the future if there is a
10057	* pressing need we can explore making this particular set of conditions work.
10058	* Right now we just cause a VM-entry failure.
10059	*
10060	* This has already been checked by VMLAUNCH/VMRESUME instruction emulation,
10061	* so should not really failure at the moment.
10062	*/
10063	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10064	}
10065	else
10066	{
10067	/*
10068	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10069	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10070	* be taken care of by EPT/shadow paging.
10071	*/
10072	if (pVM->hm.s.fAllow64BitGuests)
10073	{
10074	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10075	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10076	}
10077	}
10078
10079	/*
10080	* Validate basic assumptions.
10081	*/
10082	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10083	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10084	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10085	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10086
10087	/*
10088	* Commit it to the nested-guest VMCS.
10089	*/
10090	int rc = VINF_SUCCESS;
10091	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10092	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10093	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10094	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10095	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10096	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10097	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10098	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10099	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10100	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10101	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10102	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10103	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10104	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10105	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10106	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10107	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10108	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10109	{
10110	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10111	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10112	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10113	}
10114	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10115	{
10116	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10117	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10118	}
10119	AssertRCReturn(rc, rc);
10120
10121	/*
10122	* Update the nested-guest VMCS cache.
10123	*/
10124	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10125	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10126	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10127	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10128	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10129	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10130	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10131	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10132	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10133
10134	/*
10135	* MSR bitmap.
10136	*
10137	* The MSR bitmap address has already been initialized while setting up the
10138	* nested-guest VMCS, here we need to merge the MSR bitmaps.
10139	*/
10140	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10141	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10142
10143	return VINF_SUCCESS;
10144	}
10145	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10146
10147
10148	/**
10149	* Does the preparations before executing guest code in VT-x.
10150	*
10151	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10152	* recompiler/IEM. We must be cautious what we do here regarding committing
10153	* guest-state information into the VMCS assuming we assuredly execute the
10154	* guest in VT-x mode.
10155	*
10156	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10157	* the common-state (TRPM/forceflags), we must undo those changes so that the
10158	* recompiler/IEM can (and should) use them when it resumes guest execution.
10159	* Otherwise such operations must be done when we can no longer exit to ring-3.
10160	*
10161	* @returns Strict VBox status code (i.e. informational status codes too).
10162	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10163	* have been disabled.
10164	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10165	* double-fault into the guest.
10166	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10167	* dispatched directly.
10168	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10169	*
10170	* @param pVCpu The cross context virtual CPU structure.
10171	* @param pVmxTransient The VMX-transient structure.
10172	* @param fStepping Whether we are single-stepping the guest in the
10173	* hypervisor debugger. Makes us ignore some of the reasons
10174	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10175	* if event dispatching took place.
10176	*/
10177	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10178	{
10179	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10180
10181	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10182	if (pVmxTransient->fIsNestedGuest)
10183	{
10184	RT_NOREF2(pVCpu, fStepping);
10185	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10186	return VINF_EM_RESCHEDULE_REM;
10187	}
10188	#endif
10189
10190	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10191	PGMRZDynMapFlushAutoSet(pVCpu);
10192	#endif
10193
10194	/*
10195	* Check and process force flag actions, some of which might require us to go back to ring-3.
10196	*/
10197	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10198	if (rcStrict == VINF_SUCCESS)
10199	{ /* FFs don't get set all the time. */ }
10200	else
10201	return rcStrict;
10202
10203	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10204	/*
10205	* Switch to the nested-guest VMCS as we may have transitioned into executing
10206	* the nested-guest without leaving ring-0. Otherwise, if we came from ring-3
10207	* we would load the nested-guest VMCS while entering the VMX ring-0 session.
10208	*
10209	* We do this as late as possible to minimize (though not completely remove)
10210	* clearing/loading VMCS again due to premature trips to ring-3 above.
10211	*/
10212	if (pVmxTransient->fIsNestedGuest)
10213	{
10214	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10215	{
10216	/*
10217	* Ensure we have synced everything from the guest VMCS and also flag that
10218	* that we need to export the full (nested) guest-CPU context to the
10219	* nested-guest VMCS.
10220	*/
10221	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
10222	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST);
10223
10224	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
10225	int rc = hmR0VmxSwitchVmcs(&pVCpu->hm.s.vmx.VmcsInfo, &pVCpu->hm.s.vmx.VmcsInfoNstGst);
10226	if (RT_LIKELY(rc == VINF_SUCCESS))
10227	{
10228	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = true;
10229	ASMSetFlags(fEFlags);
10230	pVmxTransient->pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10231
10232	/*
10233	* We use a different VM-exit MSR-store area for the nested-guest. Hence,
10234	* flag that we need to update the host MSR values there. Even if we decide
10235	* in the future to share the VM-exit MSR-store area page with the guest,
10236	* if its content differs, we would have to update the host MSRs anyway.
10237	*/
10238	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
10239	Assert(!pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer); /** @todo NSTVMX: Paranoia remove later. */
10240	}
10241	else
10242	{
10243	ASMSetFlags(fEFlags);
10244	return rc;
10245	}
10246	}
10247
10248	/*
10249	* Merge guest VMCS controls with the nested-guest VMCS controls.
10250	*
10251	* Even if we have not executed the guest prior to this (e.g. when resuming
10252	* from a saved state), we should be okay with merging controls as we
10253	* initialize the guest VMCS controls as part of VM setup phase.
10254	*/
10255	if (!pVCpu->hm.s.vmx.fMergedNstGstCtls)
10256	{
10257	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10258	AssertRCReturn(rc, rc);
10259	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10260	}
10261	}
10262	#endif
10263
10264	/*
10265	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10266	* We look at the guest VMCS control here as we always set it when supported by
10267	* the physical CPU. Looking at the nested-guest control here would not be
10268	* possible because they are not merged yet.
10269	*/
10270	PVM pVM = pVCpu->CTX_SUFF(pVM);
10271	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10272	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10273	&& (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10274	&& PDMHasApic(pVM))
10275	{
10276	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10277	AssertRCReturn(rc, rc);
10278	}
10279
10280	/*
10281	* Evaluate events to be injected into the guest.
10282	*
10283	* Events in TRPM can be injected without inspecting the guest state.
10284	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10285	* guest to cause a VM-exit the next time they are ready to receive the event.
10286	*/
10287	if (TRPMHasTrap(pVCpu))
10288	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10289
10290	uint32_t fIntrState;
10291	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10292
10293	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10294	/*
10295	* While evaluating pending events if something failed (unlikely) or if we were
10296	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10297	*/
10298	if ( rcStrict != VINF_SUCCESS
10299	\|\| ( pVmxTransient->fIsNestedGuest
10300	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)))
10301	return rcStrict;
10302	#endif
10303
10304	/*
10305	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10306	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10307	* also result in triple-faulting the VM.
10308	*
10309	* The above does not apply when executing a nested-guest (since unrestricted guest execution
10310	* is a requirement) regardless doing it avoid duplicating code elsewhere.
10311	*/
10312	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10313	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10314	{ /* likely */ }
10315	else
10316	{
10317	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10318	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10319	return rcStrict;
10320	}
10321
10322	/*
10323	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10324	* import CR3 themselves. We will need to update them here, as even as late as the above
10325	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10326	* the below force flags to be set.
10327	*/
10328	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10329	{
10330	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10331	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10332	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10333	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10334	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10335	}
10336	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10337	{
10338	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10339	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10340	}
10341
10342	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10343	/* Paranoia. */
10344	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10345	#endif
10346
10347	/*
10348	* No longjmps to ring-3 from this point on!!!
10349	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10350	* This also disables flushing of the R0-logger instance (if any).
10351	*/
10352	VMMRZCallRing3Disable(pVCpu);
10353
10354	/*
10355	* Export the guest state bits.
10356	*
10357	* We cannot perform longjmps while loading the guest state because we do not preserve the
10358	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10359	* CPU migration.
10360	*
10361	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10362	* registers. Hence, loading of the guest state needs to be done -after- injection of events.
10363	*/
10364	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10365	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10366	{ /* likely */ }
10367	else
10368	{
10369	VMMRZCallRing3Enable(pVCpu);
10370	return rcStrict;
10371	}
10372
10373	/*
10374	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10375	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10376	* preemption disabled for a while. Since this is purely to aid the
10377	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10378	* disable interrupt on NT.
10379	*
10380	* We need to check for force-flags that could've possible been altered since we last
10381	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10382	* see @bugref{6398}).
10383	*
10384	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10385	* to ring-3 before executing guest code.
10386	*/
10387	pVmxTransient->fEFlags = ASMIntDisableFlags();
10388
10389	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10390	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10391	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10392	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10393	{
10394	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10395	{
10396	pVCpu->hm.s.Event.fPending = false;
10397
10398	/*
10399	* We've injected any pending events. This is really the point of no return (to ring-3).
10400	*
10401	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10402	* returns from this function, so don't enable them here.
10403	*/
10404	return VINF_SUCCESS;
10405	}
10406
10407	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10408	rcStrict = VINF_EM_RAW_INTERRUPT;
10409	}
10410	else
10411	{
10412	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10413	rcStrict = VINF_EM_RAW_TO_R3;
10414	}
10415
10416	ASMSetFlags(pVmxTransient->fEFlags);
10417	VMMRZCallRing3Enable(pVCpu);
10418
10419	return rcStrict;
10420	}
10421
10422
10423	/**
10424	* Final preparations before executing guest code using hardware-assisted VMX.
10425	*
10426	* We can no longer get preempted to a different host CPU and there are no returns
10427	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10428	* failures), this function is not intended to fail sans unrecoverable hardware
10429	* errors.
10430	*
10431	* @param pVCpu The cross context virtual CPU structure.
10432	* @param pVmxTransient The VMX-transient structure.
10433	*
10434	* @remarks Called with preemption disabled.
10435	* @remarks No-long-jump zone!!!
10436	*/
10437	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10438	{
10439	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10440	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10441	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10442	Assert(!pVCpu->hm.s.Event.fPending);
10443
10444	/*
10445	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10446	*/
10447	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10448	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10449
10450	PVM pVM = pVCpu->CTX_SUFF(pVM);
10451	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10452
10453	if (!CPUMIsGuestFPUStateActive(pVCpu))
10454	{
10455	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10456	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10457	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10458	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10459	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10460	}
10461
10462	/*
10463	* Re-save the host state bits as we may've been preempted (only happens when
10464	* thread-context hooks are used or when the VM start function changes).
10465	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10466	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10467	* see @bugref{8432}.
10468	*
10469	* This may also happen when switching to/from a nested-guest VMCS without leaving
10470	* ring-0.
10471	*/
10472	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10473	{
10474	int rc = hmR0VmxExportHostState(pVCpu);
10475	AssertRC(rc);
10476	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
10477	}
10478	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10479
10480	/*
10481	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10482	*/
10483	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10484	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10485	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10486
10487	/*
10488	* Store status of the shared guest/host debug state at the time of VM-entry.
10489	*/
10490	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
10491	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
10492	{
10493	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
10494	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
10495	}
10496	else
10497	#endif
10498	{
10499	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10500	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10501	}
10502
10503	/*
10504	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10505	* more than one conditional check. The post-run side of our code shall determine
10506	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10507	*/
10508	if (pVmcsInfo->pbVirtApic)
10509	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10510
10511	/*
10512	* Update the host MSRs values in the VM-exit MSR-load area.
10513	*/
10514	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10515	{
10516	if (pVmcsInfo->cExitMsrLoad > 0)
10517	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10518	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10519	}
10520
10521	/*
10522	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10523	* VMX-preemption timer based on the next virtual sync clock deadline.
10524	*/
10525	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10526	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10527	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10528	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10529	{
10530	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10531	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10532	}
10533
10534	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10535	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10536	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10537	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
10538
10539	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10540
10541	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10542	as we're about to start executing the guest . */
10543
10544	/*
10545	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10546	*
10547	* This is done this late as updating the TSC offsetting/preemption timer above
10548	* figures out if we can skip intercepting RDTSCP by calculating the number of
10549	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10550	*/
10551	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10552	{
10553	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10554	{
10555	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10556	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10557	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10558	AssertRC(rc);
10559	}
10560	else
10561	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
10562	}
10563
10564	#ifdef VBOX_STRICT
10565	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo);
10566	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10567	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo));
10568	#endif
10569
10570	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10571	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10572	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs. */
10573	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10574	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10575	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10576	#endif
10577	}
10578
10579
10580	/**
10581	* First C routine invoked after running guest code using hardware-assisted VMX.
10582	*
10583	* @param pVCpu The cross context virtual CPU structure.
10584	* @param pVmxTransient The VMX-transient structure.
10585	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10586	*
10587	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10588	*
10589	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10590	* unconditionally when it is safe to do so.
10591	*/
10592	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10593	{
10594	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10595
10596	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10597	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10598	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10599	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10600	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10601	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10602
10603	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10604	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10605	{
10606	uint64_t uGstTsc;
10607	if (!pVmxTransient->fIsNestedGuest)
10608	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10609	else
10610	{
10611	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10612	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10613	}
10614	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10615	}
10616
10617	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10618	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
10619	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10620
10621	#if HC_ARCH_BITS == 64
10622	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10623	#endif
10624	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
10625	/* The 64-on-32 switcher maintains VMCS-launch state on its own
10626	and we need to leave it alone here. */
10627	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
10628	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10629	#else
10630	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10631	#endif
10632	#ifdef VBOX_STRICT
10633	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10634	#endif
10635	Assert(!ASMIntAreEnabled());
10636	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10637	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10638
10639	/*
10640	* Save the basic VM-exit reason and check if the VM-entry failed.
10641	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10642	*/
10643	uint32_t uExitReason;
10644	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
10645	AssertRC(rc);
10646	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
10647	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
10648
10649	/*
10650	* Check if VMLAUNCH/VMRESUME succeeded.
10651	* If this failed, we cause a guru meditation and cease further execution.
10652	*/
10653	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
10654	{
10655	/*
10656	* Update the VM-exit history array here even if the VM-entry failed due to:
10657	* - Invalid guest state.
10658	* - MSR loading.
10659	* - Machine-check event.
10660	*
10661	* In any of the above cases we will still have a "valid" VM-exit reason
10662	* despite @a fVMEntryFailed being false.
10663	*
10664	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
10665	*
10666	* Note! We don't have CS or RIP at this point. Will probably address that later
10667	* by amending the history entry added here.
10668	*/
10669	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
10670	UINT64_MAX, uHostTsc);
10671
10672	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
10673	{
10674	VMMRZCallRing3Enable(pVCpu);
10675
10676	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10677	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10678
10679	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
10680	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10681	AssertRC(rc);
10682	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
10683	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
10684	AssertRC(rc);
10685	#else
10686	/*
10687	* Import the guest-interruptibility state always as we need it while evaluating
10688	* injecting events on re-entry.
10689	*
10690	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
10691	* checking for real-mode while exporting the state because all bits that cause
10692	* mode changes wrt CR0 are intercepted.
10693	*/
10694	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
10695	AssertRC(rc);
10696	#endif
10697
10698	/*
10699	* Sync the TPR shadow with our APIC state.
10700	*/
10701	if ( !pVmxTransient->fIsNestedGuest
10702	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
10703	{
10704	Assert(pVmcsInfo->pbVirtApic);
10705	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
10706	{
10707	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
10708	AssertRC(rc);
10709	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
10710	}
10711	}
10712
10713	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10714	return;
10715	}
10716	}
10717	else
10718	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
10719
10720	VMMRZCallRing3Enable(pVCpu);
10721	}
10722
10723
10724	/**
10725	* Runs the guest code using hardware-assisted VMX the normal way.
10726	*
10727	* @returns VBox status code.
10728	* @param pVCpu The cross context virtual CPU structure.
10729	* @param pcLoops Pointer to the number of executed loops.
10730	*/
10731	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
10732	{
10733	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10734	Assert(pcLoops);
10735	Assert(*pcLoops <= cMaxResumeLoops);
10736
10737	VMXTRANSIENT VmxTransient;
10738	RT_ZERO(VmxTransient);
10739	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10740
10741	/* Paranoia. */
10742	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
10743	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10744
10745	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10746	for (;;)
10747	{
10748	Assert(!HMR0SuspendPending());
10749	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10750	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10751
10752	/*
10753	* Preparatory work for running nested-guest code, this may force us to
10754	* return to ring-3.
10755	*
10756	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10757	*/
10758	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10759	if (rcStrict != VINF_SUCCESS)
10760	break;
10761
10762	/* Interrupts are disabled at this point! */
10763	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10764	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10765	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10766	/* Interrupts are re-enabled at this point! */
10767
10768	/*
10769	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10770	*/
10771	if (RT_SUCCESS(rcRun))
10772	{ /* very likely */ }
10773	else
10774	{
10775	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10776	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10777	return rcRun;
10778	}
10779
10780	/*
10781	* Profile the VM-exit.
10782	*/
10783	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10784	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10785	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10786	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10787	HMVMX_START_EXIT_DISPATCH_PROF();
10788
10789	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10790
10791	/*
10792	* Handle the VM-exit.
10793	*/
10794	#ifdef HMVMX_USE_FUNCTION_TABLE
10795	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
10796	#else
10797	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
10798	#endif
10799	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10800	if (rcStrict == VINF_SUCCESS)
10801	{
10802	if (++(*pcLoops) <= cMaxResumeLoops)
10803	continue;
10804	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10805	rcStrict = VINF_EM_RAW_INTERRUPT;
10806	}
10807	break;
10808	}
10809
10810	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10811	return rcStrict;
10812	}
10813
10814	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10815	/**
10816	* Runs the nested-guest code using hardware-assisted VMX.
10817	*
10818	* @returns VBox status code.
10819	* @param pVCpu The cross context virtual CPU structure.
10820	* @param pcLoops Pointer to the number of executed loops.
10821	*
10822	* @sa hmR0VmxRunGuestCodeNormal().
10823	*/
10824	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
10825	{
10826	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10827	Assert(pcLoops);
10828	Assert(*pcLoops <= cMaxResumeLoops);
10829
10830	VMXTRANSIENT VmxTransient;
10831	RT_ZERO(VmxTransient);
10832	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10833	VmxTransient.fIsNestedGuest = true;
10834
10835	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10836	for (;;)
10837	{
10838	Assert(!HMR0SuspendPending());
10839	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10840	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10841
10842	/*
10843	* Preparatory work for running guest code, this may force us to
10844	* return to ring-3.
10845	*
10846	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10847	*/
10848	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10849	if (rcStrict != VINF_SUCCESS)
10850	break;
10851
10852	/* Interrupts are disabled at this point! */
10853	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10854	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10855	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10856	/* Interrupts are re-enabled at this point! */
10857
10858	/*
10859	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10860	*/
10861	if (RT_SUCCESS(rcRun))
10862	{ /* very likely */ }
10863	else
10864	{
10865	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10866	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10867	return rcRun;
10868	}
10869
10870	/*
10871	* Profile the VM-exit.
10872	*/
10873	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10874	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10875	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10876	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10877	HMVMX_START_EXIT_DISPATCH_PROF();
10878
10879	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10880
10881	/*
10882	* Handle the VM-exit.
10883	*/
10884	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
10885	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10886	if ( rcStrict == VINF_SUCCESS
10887	&& CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10888	{
10889	if (++(*pcLoops) <= cMaxResumeLoops)
10890	continue;
10891	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10892	rcStrict = VINF_EM_RAW_INTERRUPT;
10893	}
10894	break;
10895	}
10896
10897	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10898	return rcStrict;
10899	}
10900	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10901
10902
10903	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
10904	* probes.
10905	*
10906	* The following few functions and associated structure contains the bloat
10907	* necessary for providing detailed debug events and dtrace probes as well as
10908	* reliable host side single stepping. This works on the principle of
10909	* "subclassing" the normal execution loop and workers. We replace the loop
10910	* method completely and override selected helpers to add necessary adjustments
10911	* to their core operation.
10912	*
10913	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
10914	* any performance for debug and analysis features.
10915	*
10916	* @{
10917	*/
10918
10919	/**
10920	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
10921	* the debug run loop.
10922	*/
10923	typedef struct VMXRUNDBGSTATE
10924	{
10925	/** The RIP we started executing at. This is for detecting that we stepped. */
10926	uint64_t uRipStart;
10927	/** The CS we started executing with. */
10928	uint16_t uCsStart;
10929
10930	/** Whether we've actually modified the 1st execution control field. */
10931	bool fModifiedProcCtls : 1;
10932	/** Whether we've actually modified the 2nd execution control field. */
10933	bool fModifiedProcCtls2 : 1;
10934	/** Whether we've actually modified the exception bitmap. */
10935	bool fModifiedXcptBitmap : 1;
10936
10937	/** We desire the modified the CR0 mask to be cleared. */
10938	bool fClearCr0Mask : 1;
10939	/** We desire the modified the CR4 mask to be cleared. */
10940	bool fClearCr4Mask : 1;
10941	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
10942	uint32_t fCpe1Extra;
10943	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
10944	uint32_t fCpe1Unwanted;
10945	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
10946	uint32_t fCpe2Extra;
10947	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
10948	uint32_t bmXcptExtra;
10949	/** The sequence number of the Dtrace provider settings the state was
10950	* configured against. */
10951	uint32_t uDtraceSettingsSeqNo;
10952	/** VM-exits to check (one bit per VM-exit). */
10953	uint32_t bmExitsToCheck[3];
10954
10955	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
10956	uint32_t fProcCtlsInitial;
10957	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
10958	uint32_t fProcCtls2Initial;
10959	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
10960	uint32_t bmXcptInitial;
10961	} VMXRUNDBGSTATE;
10962	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
10963	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
10964
10965
10966	/**
10967	* Initializes the VMXRUNDBGSTATE structure.
10968	*
10969	* @param pVCpu The cross context virtual CPU structure of the
10970	* calling EMT.
10971	* @param pVmxTransient The VMX-transient structure.
10972	* @param pDbgState The debug state to initialize.
10973	*/
10974	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
10975	{
10976	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
10977	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
10978
10979	pDbgState->fModifiedProcCtls = false;
10980	pDbgState->fModifiedProcCtls2 = false;
10981	pDbgState->fModifiedXcptBitmap = false;
10982	pDbgState->fClearCr0Mask = false;
10983	pDbgState->fClearCr4Mask = false;
10984	pDbgState->fCpe1Extra = 0;
10985	pDbgState->fCpe1Unwanted = 0;
10986	pDbgState->fCpe2Extra = 0;
10987	pDbgState->bmXcptExtra = 0;
10988	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
10989	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
10990	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
10991	}
10992
10993
10994	/**
10995	* Updates the VMSC fields with changes requested by @a pDbgState.
10996	*
10997	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
10998	* immediately before executing guest code, i.e. when interrupts are disabled.
10999	* We don't check status codes here as we cannot easily assert or return in the
11000	* latter case.
11001	*
11002	* @param pVCpu The cross context virtual CPU structure.
11003	* @param pVmxTransient The VMX-transient structure.
11004	* @param pDbgState The debug state.
11005	*/
11006	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11007	{
11008	/*
11009	* Ensure desired flags in VMCS control fields are set.
11010	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11011	*
11012	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11013	* there should be no stale data in pCtx at this point.
11014	*/
11015	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11016	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11017	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11018	{
11019	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11020	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11021	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11022	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11023	pDbgState->fModifiedProcCtls = true;
11024	}
11025
11026	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11027	{
11028	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11029	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11030	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11031	pDbgState->fModifiedProcCtls2 = true;
11032	}
11033
11034	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11035	{
11036	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11037	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11038	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11039	pDbgState->fModifiedXcptBitmap = true;
11040	}
11041
11042	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11043	{
11044	pVmcsInfo->u64Cr0Mask = 0;
11045	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
11046	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11047	}
11048
11049	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11050	{
11051	pVmcsInfo->u64Cr4Mask = 0;
11052	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
11053	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11054	}
11055
11056	NOREF(pVCpu);
11057	}
11058
11059
11060	/**
11061	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11062	* re-entry next time around.
11063	*
11064	* @returns Strict VBox status code (i.e. informational status codes too).
11065	* @param pVCpu The cross context virtual CPU structure.
11066	* @param pVmxTransient The VMX-transient structure.
11067	* @param pDbgState The debug state.
11068	* @param rcStrict The return code from executing the guest using single
11069	* stepping.
11070	*/
11071	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11072	VBOXSTRICTRC rcStrict)
11073	{
11074	/*
11075	* Restore VM-exit control settings as we may not reenter this function the
11076	* next time around.
11077	*/
11078	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11079
11080	/* We reload the initial value, trigger what we can of recalculations the
11081	next time around. From the looks of things, that's all that's required atm. */
11082	if (pDbgState->fModifiedProcCtls)
11083	{
11084	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11085	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11086	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11087	AssertRCReturn(rc2, rc2);
11088	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11089	}
11090
11091	/* We're currently the only ones messing with this one, so just restore the
11092	cached value and reload the field. */
11093	if ( pDbgState->fModifiedProcCtls2
11094	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11095	{
11096	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11097	AssertRCReturn(rc2, rc2);
11098	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11099	}
11100
11101	/* If we've modified the exception bitmap, we restore it and trigger
11102	reloading and partial recalculation the next time around. */
11103	if (pDbgState->fModifiedXcptBitmap)
11104	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11105
11106	return rcStrict;
11107	}
11108
11109
11110	/**
11111	* Configures VM-exit controls for current DBGF and DTrace settings.
11112	*
11113	* This updates @a pDbgState and the VMCS execution control fields to reflect
11114	* the necessary VM-exits demanded by DBGF and DTrace.
11115	*
11116	* @param pVCpu The cross context virtual CPU structure.
11117	* @param pVmxTransient The VMX-transient structure. May update
11118	* fUpdatedTscOffsettingAndPreemptTimer.
11119	* @param pDbgState The debug state.
11120	*/
11121	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11122	{
11123	/*
11124	* Take down the dtrace serial number so we can spot changes.
11125	*/
11126	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11127	ASMCompilerBarrier();
11128
11129	/*
11130	* We'll rebuild most of the middle block of data members (holding the
11131	* current settings) as we go along here, so start by clearing it all.
11132	*/
11133	pDbgState->bmXcptExtra = 0;
11134	pDbgState->fCpe1Extra = 0;
11135	pDbgState->fCpe1Unwanted = 0;
11136	pDbgState->fCpe2Extra = 0;
11137	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11138	pDbgState->bmExitsToCheck[i] = 0;
11139
11140	/*
11141	* Software interrupts (INT XXh) - no idea how to trigger these...
11142	*/
11143	PVM pVM = pVCpu->CTX_SUFF(pVM);
11144	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11145	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11146	{
11147	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11148	}
11149
11150	/*
11151	* INT3 breakpoints - triggered by #BP exceptions.
11152	*/
11153	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11154	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11155
11156	/*
11157	* Exception bitmap and XCPT events+probes.
11158	*/
11159	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11160	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11161	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11162
11163	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11164	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11165	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11166	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11167	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11168	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11169	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11170	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11171	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11172	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11173	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11174	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11175	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11176	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11177	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11178	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11179	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11180	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11181
11182	if (pDbgState->bmXcptExtra)
11183	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11184
11185	/*
11186	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11187	*
11188	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11189	* So, when adding/changing/removing please don't forget to update it.
11190	*
11191	* Some of the macros are picking up local variables to save horizontal space,
11192	* (being able to see it in a table is the lesser evil here).
11193	*/
11194	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11195	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11196	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11197	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11198	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11199	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11200	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11201	} else do { } while (0)
11202	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11203	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11204	{ \
11205	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11206	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11207	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11208	} else do { } while (0)
11209	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11210	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11211	{ \
11212	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11213	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11214	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11215	} else do { } while (0)
11216	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11217	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11218	{ \
11219	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11220	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11221	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11222	} else do { } while (0)
11223
11224	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11225	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11226	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11227	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11228	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11229
11230	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11231	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11232	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11233	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11234	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11235	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11236	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11237	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11238	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11239	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11240	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11241	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11242	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11243	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11244	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11245	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11246	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11247	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11248	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11249	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11250	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11251	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11252	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11253	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11254	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11255	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11256	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11257	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11258	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11259	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11260	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11261	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11262	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11263	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11264	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11265	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11266
11267	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11268	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11269	{
11270	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11271	\| CPUMCTX_EXTRN_APIC_TPR);
11272	AssertRC(rc);
11273
11274	#if 0 /** @todo fix me */
11275	pDbgState->fClearCr0Mask = true;
11276	pDbgState->fClearCr4Mask = true;
11277	#endif
11278	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11279	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11280	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11281	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11282	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11283	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11284	require clearing here and in the loop if we start using it. */
11285	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11286	}
11287	else
11288	{
11289	if (pDbgState->fClearCr0Mask)
11290	{
11291	pDbgState->fClearCr0Mask = false;
11292	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11293	}
11294	if (pDbgState->fClearCr4Mask)
11295	{
11296	pDbgState->fClearCr4Mask = false;
11297	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11298	}
11299	}
11300	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11301	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11302
11303	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11304	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11305	{
11306	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11307	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11308	}
11309	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11310	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11311
11312	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11313	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11314	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11315	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11316	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11317	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11318	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11319	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11320	#if 0 /** @todo too slow, fix handler. */
11321	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11322	#endif
11323	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11324
11325	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11326	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11327	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11328	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11329	{
11330	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11331	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11332	}
11333	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11334	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11335	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11336	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11337
11338	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11339	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11340	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11341	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11342	{
11343	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11344	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11345	}
11346	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11347	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11348	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11349	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11350
11351	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11352	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11353	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11354	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11355	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11356	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11357	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11358	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11359	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11360	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11361	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11362	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11363	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11364	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11365	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11366	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11367	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11368	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11369	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11370	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11371	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11372	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11373
11374	#undef IS_EITHER_ENABLED
11375	#undef SET_ONLY_XBM_IF_EITHER_EN
11376	#undef SET_CPE1_XBM_IF_EITHER_EN
11377	#undef SET_CPEU_XBM_IF_EITHER_EN
11378	#undef SET_CPE2_XBM_IF_EITHER_EN
11379
11380	/*
11381	* Sanitize the control stuff.
11382	*/
11383	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11384	if (pDbgState->fCpe2Extra)
11385	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11386	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11387	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11388	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11389	{
11390	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11391	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11392	}
11393
11394	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11395	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11396	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11397	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11398	}
11399
11400
11401	/**
11402	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11403	* appropriate.
11404	*
11405	* The caller has checked the VM-exit against the
11406	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11407	* already, so we don't have to do that either.
11408	*
11409	* @returns Strict VBox status code (i.e. informational status codes too).
11410	* @param pVCpu The cross context virtual CPU structure.
11411	* @param pVmxTransient The VMX-transient structure.
11412	* @param uExitReason The VM-exit reason.
11413	*
11414	* @remarks The name of this function is displayed by dtrace, so keep it short
11415	* and to the point. No longer than 33 chars long, please.
11416	*/
11417	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11418	{
11419	/*
11420	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11421	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11422	*
11423	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11424	* does. Must add/change/remove both places. Same ordering, please.
11425	*
11426	* Added/removed events must also be reflected in the next section
11427	* where we dispatch dtrace events.
11428	*/
11429	bool fDtrace1 = false;
11430	bool fDtrace2 = false;
11431	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11432	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11433	uint32_t uEventArg = 0;
11434	#define SET_EXIT(a_EventSubName) \
11435	do { \
11436	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11437	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11438	} while (0)
11439	#define SET_BOTH(a_EventSubName) \
11440	do { \
11441	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11442	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11443	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11444	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11445	} while (0)
11446	switch (uExitReason)
11447	{
11448	case VMX_EXIT_MTF:
11449	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11450
11451	case VMX_EXIT_XCPT_OR_NMI:
11452	{
11453	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11454	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11455	{
11456	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11457	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11458	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11459	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11460	{
11461	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11462	{
11463	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11464	uEventArg = pVmxTransient->uExitIntErrorCode;
11465	}
11466	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11467	switch (enmEvent1)
11468	{
11469	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11470	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11471	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11472	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11473	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11474	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11475	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11476	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11477	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11478	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11479	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11480	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11481	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11482	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11483	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11484	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11485	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11486	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11487	default: break;
11488	}
11489	}
11490	else
11491	AssertFailed();
11492	break;
11493
11494	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11495	uEventArg = idxVector;
11496	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11497	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11498	break;
11499	}
11500	break;
11501	}
11502
11503	case VMX_EXIT_TRIPLE_FAULT:
11504	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11505	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11506	break;
11507	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11508	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11509	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11510	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11511	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11512
11513	/* Instruction specific VM-exits: */
11514	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11515	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11516	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11517	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11518	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11519	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11520	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11521	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11522	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11523	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11524	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11525	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11526	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11527	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11528	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11529	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11530	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11531	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11532	case VMX_EXIT_MOV_CRX:
11533	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11534	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11535	SET_BOTH(CRX_READ);
11536	else
11537	SET_BOTH(CRX_WRITE);
11538	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11539	break;
11540	case VMX_EXIT_MOV_DRX:
11541	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11542	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11543	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11544	SET_BOTH(DRX_READ);
11545	else
11546	SET_BOTH(DRX_WRITE);
11547	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11548	break;
11549	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11550	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11551	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11552	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11553	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11554	case VMX_EXIT_GDTR_IDTR_ACCESS:
11555	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11556	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11557	{
11558	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11559	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11560	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11561	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11562	}
11563	break;
11564
11565	case VMX_EXIT_LDTR_TR_ACCESS:
11566	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11567	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11568	{
11569	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11570	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11571	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11572	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11573	}
11574	break;
11575
11576	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11577	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11578	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11579	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11580	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11581	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11582	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11583	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11584	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11585	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11586	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11587
11588	/* Events that aren't relevant at this point. */
11589	case VMX_EXIT_EXT_INT:
11590	case VMX_EXIT_INT_WINDOW:
11591	case VMX_EXIT_NMI_WINDOW:
11592	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11593	case VMX_EXIT_PREEMPT_TIMER:
11594	case VMX_EXIT_IO_INSTR:
11595	break;
11596
11597	/* Errors and unexpected events. */
11598	case VMX_EXIT_INIT_SIGNAL:
11599	case VMX_EXIT_SIPI:
11600	case VMX_EXIT_IO_SMI:
11601	case VMX_EXIT_SMI:
11602	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11603	case VMX_EXIT_ERR_MSR_LOAD:
11604	case VMX_EXIT_ERR_MACHINE_CHECK:
11605	break;
11606
11607	default:
11608	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11609	break;
11610	}
11611	#undef SET_BOTH
11612	#undef SET_EXIT
11613
11614	/*
11615	* Dtrace tracepoints go first. We do them here at once so we don't
11616	* have to copy the guest state saving and stuff a few dozen times.
11617	* Down side is that we've got to repeat the switch, though this time
11618	* we use enmEvent since the probes are a subset of what DBGF does.
11619	*/
11620	if (fDtrace1 \|\| fDtrace2)
11621	{
11622	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11623	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11624	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11625	switch (enmEvent1)
11626	{
11627	/** @todo consider which extra parameters would be helpful for each probe. */
11628	case DBGFEVENT_END: break;
11629	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11630	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11631	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11632	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11633	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11634	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11635	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11636	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11637	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
11638	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
11639	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
11640	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
11641	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
11642	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
11643	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
11644	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
11645	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
11646	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
11647	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11648	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11649	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
11650	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
11651	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
11652	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
11653	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
11654	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
11655	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
11656	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11657	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11658	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11659	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11660	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11661	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
11662	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11663	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
11664	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
11665	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
11666	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
11667	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
11668	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
11669	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
11670	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
11671	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
11672	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
11673	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
11674	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
11675	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
11676	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
11677	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
11678	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
11679	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
11680	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
11681	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
11682	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
11683	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
11684	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
11685	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
11686	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
11687	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
11688	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
11689	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
11690	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
11691	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
11692	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
11693	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
11694	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
11695	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
11696	}
11697	switch (enmEvent2)
11698	{
11699	/** @todo consider which extra parameters would be helpful for each probe. */
11700	case DBGFEVENT_END: break;
11701	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
11702	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11703	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
11704	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
11705	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
11706	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
11707	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
11708	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
11709	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
11710	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11711	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11712	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11713	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11714	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11715	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
11716	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11717	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
11718	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
11719	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
11720	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
11721	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
11722	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
11723	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
11724	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
11725	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
11726	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
11727	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
11728	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
11729	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
11730	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
11731	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
11732	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
11733	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
11734	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
11735	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
11736	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
11737	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
11738	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
11739	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
11740	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
11741	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
11742	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
11743	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
11744	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
11745	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
11746	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
11747	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
11748	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
11749	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
11750	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
11751	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
11752	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
11753	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
11754	}
11755	}
11756
11757	/*
11758	* Fire of the DBGF event, if enabled (our check here is just a quick one,
11759	* the DBGF call will do a full check).
11760	*
11761	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
11762	* Note! If we have to events, we prioritize the first, i.e. the instruction
11763	* one, in order to avoid event nesting.
11764	*/
11765	PVM pVM = pVCpu->CTX_SUFF(pVM);
11766	if ( enmEvent1 != DBGFEVENT_END
11767	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
11768	{
11769	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11770	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
11771	if (rcStrict != VINF_SUCCESS)
11772	return rcStrict;
11773	}
11774	else if ( enmEvent2 != DBGFEVENT_END
11775	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
11776	{
11777	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11778	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
11779	if (rcStrict != VINF_SUCCESS)
11780	return rcStrict;
11781	}
11782
11783	return VINF_SUCCESS;
11784	}
11785
11786
11787	/**
11788	* Single-stepping VM-exit filtering.
11789	*
11790	* This is preprocessing the VM-exits and deciding whether we've gotten far
11791	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
11792	* handling is performed.
11793	*
11794	* @returns Strict VBox status code (i.e. informational status codes too).
11795	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
11796	* @param pVmxTransient The VMX-transient structure.
11797	* @param pDbgState The debug state.
11798	*/
11799	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11800	{
11801	/*
11802	* Expensive (saves context) generic dtrace VM-exit probe.
11803	*/
11804	uint32_t const uExitReason = pVmxTransient->uExitReason;
11805	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
11806	{ /* more likely */ }
11807	else
11808	{
11809	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11810	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11811	AssertRC(rc);
11812	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
11813	}
11814
11815	/*
11816	* Check for host NMI, just to get that out of the way.
11817	*/
11818	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
11819	{ /* normally likely */ }
11820	else
11821	{
11822	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11823	AssertRCReturn(rc2, rc2);
11824	uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11825	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11826	return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
11827	}
11828
11829	/*
11830	* Check for single stepping event if we're stepping.
11831	*/
11832	if (pVCpu->hm.s.fSingleInstruction)
11833	{
11834	switch (uExitReason)
11835	{
11836	case VMX_EXIT_MTF:
11837	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11838
11839	/* Various events: */
11840	case VMX_EXIT_XCPT_OR_NMI:
11841	case VMX_EXIT_EXT_INT:
11842	case VMX_EXIT_TRIPLE_FAULT:
11843	case VMX_EXIT_INT_WINDOW:
11844	case VMX_EXIT_NMI_WINDOW:
11845	case VMX_EXIT_TASK_SWITCH:
11846	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11847	case VMX_EXIT_APIC_ACCESS:
11848	case VMX_EXIT_EPT_VIOLATION:
11849	case VMX_EXIT_EPT_MISCONFIG:
11850	case VMX_EXIT_PREEMPT_TIMER:
11851
11852	/* Instruction specific VM-exits: */
11853	case VMX_EXIT_CPUID:
11854	case VMX_EXIT_GETSEC:
11855	case VMX_EXIT_HLT:
11856	case VMX_EXIT_INVD:
11857	case VMX_EXIT_INVLPG:
11858	case VMX_EXIT_RDPMC:
11859	case VMX_EXIT_RDTSC:
11860	case VMX_EXIT_RSM:
11861	case VMX_EXIT_VMCALL:
11862	case VMX_EXIT_VMCLEAR:
11863	case VMX_EXIT_VMLAUNCH:
11864	case VMX_EXIT_VMPTRLD:
11865	case VMX_EXIT_VMPTRST:
11866	case VMX_EXIT_VMREAD:
11867	case VMX_EXIT_VMRESUME:
11868	case VMX_EXIT_VMWRITE:
11869	case VMX_EXIT_VMXOFF:
11870	case VMX_EXIT_VMXON:
11871	case VMX_EXIT_MOV_CRX:
11872	case VMX_EXIT_MOV_DRX:
11873	case VMX_EXIT_IO_INSTR:
11874	case VMX_EXIT_RDMSR:
11875	case VMX_EXIT_WRMSR:
11876	case VMX_EXIT_MWAIT:
11877	case VMX_EXIT_MONITOR:
11878	case VMX_EXIT_PAUSE:
11879	case VMX_EXIT_GDTR_IDTR_ACCESS:
11880	case VMX_EXIT_LDTR_TR_ACCESS:
11881	case VMX_EXIT_INVEPT:
11882	case VMX_EXIT_RDTSCP:
11883	case VMX_EXIT_INVVPID:
11884	case VMX_EXIT_WBINVD:
11885	case VMX_EXIT_XSETBV:
11886	case VMX_EXIT_RDRAND:
11887	case VMX_EXIT_INVPCID:
11888	case VMX_EXIT_VMFUNC:
11889	case VMX_EXIT_RDSEED:
11890	case VMX_EXIT_XSAVES:
11891	case VMX_EXIT_XRSTORS:
11892	{
11893	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11894	AssertRCReturn(rc, rc);
11895	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
11896	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
11897	return VINF_EM_DBG_STEPPED;
11898	break;
11899	}
11900
11901	/* Errors and unexpected events: */
11902	case VMX_EXIT_INIT_SIGNAL:
11903	case VMX_EXIT_SIPI:
11904	case VMX_EXIT_IO_SMI:
11905	case VMX_EXIT_SMI:
11906	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11907	case VMX_EXIT_ERR_MSR_LOAD:
11908	case VMX_EXIT_ERR_MACHINE_CHECK:
11909	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
11910	break;
11911
11912	default:
11913	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11914	break;
11915	}
11916	}
11917
11918	/*
11919	* Check for debugger event breakpoints and dtrace probes.
11920	*/
11921	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
11922	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
11923	{
11924	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
11925	if (rcStrict != VINF_SUCCESS)
11926	return rcStrict;
11927	}
11928
11929	/*
11930	* Normal processing.
11931	*/
11932	#ifdef HMVMX_USE_FUNCTION_TABLE
11933	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
11934	#else
11935	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
11936	#endif
11937	}
11938
11939
11940	/**
11941	* Single steps guest code using hardware-assisted VMX.
11942	*
11943	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
11944	* but single-stepping through the hypervisor debugger.
11945	*
11946	* @returns Strict VBox status code (i.e. informational status codes too).
11947	* @param pVCpu The cross context virtual CPU structure.
11948	* @param pcLoops Pointer to the number of executed loops.
11949	*
11950	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
11951	*/
11952	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
11953	{
11954	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11955	Assert(pcLoops);
11956	Assert(*pcLoops <= cMaxResumeLoops);
11957
11958	VMXTRANSIENT VmxTransient;
11959	RT_ZERO(VmxTransient);
11960	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11961
11962	/* Set HMCPU indicators. */
11963	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
11964	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
11965	pVCpu->hm.s.fDebugWantRdTscExit = false;
11966	pVCpu->hm.s.fUsingDebugLoop = true;
11967
11968	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
11969	VMXRUNDBGSTATE DbgState;
11970	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
11971	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
11972
11973	/*
11974	* The loop.
11975	*/
11976	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11977	for (;;)
11978	{
11979	Assert(!HMR0SuspendPending());
11980	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11981	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11982	bool fStepping = pVCpu->hm.s.fSingleInstruction;
11983
11984	/* Set up VM-execution controls the next two can respond to. */
11985	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
11986
11987	/*
11988	* Preparatory work for running guest code, this may force us to
11989	* return to ring-3.
11990	*
11991	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11992	*/
11993	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
11994	if (rcStrict != VINF_SUCCESS)
11995	break;
11996
11997	/* Interrupts are disabled at this point! */
11998	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11999
12000	/* Override any obnoxious code in the above two calls. */
12001	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12002
12003	/*
12004	* Finally execute the guest.
12005	*/
12006	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12007
12008	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12009	/* Interrupts are re-enabled at this point! */
12010
12011	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12012	if (RT_SUCCESS(rcRun))
12013	{ /* very likely */ }
12014	else
12015	{
12016	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12017	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12018	return rcRun;
12019	}
12020
12021	/* Profile the VM-exit. */
12022	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12023	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12024	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12025	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12026	HMVMX_START_EXIT_DISPATCH_PROF();
12027
12028	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12029
12030	/*
12031	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12032	*/
12033	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12034	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12035	if (rcStrict != VINF_SUCCESS)
12036	break;
12037	if (++(*pcLoops) > cMaxResumeLoops)
12038	{
12039	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12040	rcStrict = VINF_EM_RAW_INTERRUPT;
12041	break;
12042	}
12043
12044	/*
12045	* Stepping: Did the RIP change, if so, consider it a single step.
12046	* Otherwise, make sure one of the TFs gets set.
12047	*/
12048	if (fStepping)
12049	{
12050	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12051	AssertRC(rc);
12052	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12053	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12054	{
12055	rcStrict = VINF_EM_DBG_STEPPED;
12056	break;
12057	}
12058	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12059	}
12060
12061	/*
12062	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12063	*/
12064	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12065	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12066	}
12067
12068	/*
12069	* Clear the X86_EFL_TF if necessary.
12070	*/
12071	if (pVCpu->hm.s.fClearTrapFlag)
12072	{
12073	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12074	AssertRC(rc);
12075	pVCpu->hm.s.fClearTrapFlag = false;
12076	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12077	}
12078	/** @todo there seems to be issues with the resume flag when the monitor trap
12079	* flag is pending without being used. Seen early in bios init when
12080	* accessing APIC page in protected mode. */
12081
12082	/*
12083	* Restore VM-exit control settings as we may not re-enter this function the
12084	* next time around.
12085	*/
12086	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12087
12088	/* Restore HMCPU indicators. */
12089	pVCpu->hm.s.fUsingDebugLoop = false;
12090	pVCpu->hm.s.fDebugWantRdTscExit = false;
12091	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12092
12093	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12094	return rcStrict;
12095	}
12096
12097
12098	/** @} */
12099
12100
12101	/**
12102	* Checks if any expensive dtrace probes are enabled and we should go to the
12103	* debug loop.
12104	*
12105	* @returns true if we should use debug loop, false if not.
12106	*/
12107	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12108	{
12109	/* It's probably faster to OR the raw 32-bit counter variables together.
12110	Since the variables are in an array and the probes are next to one
12111	another (more or less), we have good locality. So, better read
12112	eight-nine cache lines ever time and only have one conditional, than
12113	128+ conditionals, right? */
12114	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12115	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12116	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12117	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12118	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12119	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12120	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12121	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12122	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12123	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12124	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12125	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12126	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12127	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12128	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12129	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12130	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12131	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12132	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12133	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12134	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12135	) != 0
12136	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12137	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12138	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12139	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12140	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12141	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12142	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12143	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12144	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12145	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12146	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12147	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12148	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12149	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12150	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12151	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12152	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12153	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12154	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12155	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12156	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12157	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12158	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12159	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12160	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12161	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12162	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12163	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12164	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12165	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12166	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12167	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12168	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12169	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12170	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12171	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12172	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12173	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12174	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12175	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12176	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12177	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12178	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12179	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12180	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12181	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12182	) != 0
12183	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12184	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12185	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12186	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12187	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12188	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12189	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12190	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12191	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12192	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12193	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12194	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12195	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12196	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12197	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12198	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12199	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12200	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12201	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12202	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12203	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12204	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12205	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12206	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12207	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12208	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12209	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12210	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12211	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12212	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12213	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12214	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12215	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12216	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12217	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12218	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12219	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12220	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12221	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12222	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12223	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12224	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12225	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12226	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12227	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12228	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12229	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12230	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12231	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12232	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12233	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12234	) != 0;
12235	}
12236
12237
12238	/**
12239	* Runs the guest using hardware-assisted VMX.
12240	*
12241	* @returns Strict VBox status code (i.e. informational status codes too).
12242	* @param pVCpu The cross context virtual CPU structure.
12243	*/
12244	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
12245	{
12246	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12247	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12248	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12249	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12250
12251	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
12252
12253	VBOXSTRICTRC rcStrict;
12254	uint32_t cLoops = 0;
12255	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12256	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12257	#else
12258	bool const fInNestedGuestMode = false;
12259	#endif
12260	if (!fInNestedGuestMode)
12261	{
12262	if ( !pVCpu->hm.s.fUseDebugLoop
12263	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12264	&& !DBGFIsStepping(pVCpu)
12265	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12266	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12267	else
12268	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12269	}
12270	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12271	else
12272	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
12273
12274	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12275	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12276	#endif
12277
12278	if (rcStrict == VERR_EM_INTERPRETER)
12279	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12280	else if (rcStrict == VINF_EM_RESET)
12281	rcStrict = VINF_EM_TRIPLE_FAULT;
12282
12283	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12284	if (RT_FAILURE(rc2))
12285	{
12286	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12287	rcStrict = rc2;
12288	}
12289	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12290	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12291	return rcStrict;
12292	}
12293
12294
12295	#ifndef HMVMX_USE_FUNCTION_TABLE
12296	/**
12297	* Handles a guest VM-exit from hardware-assisted VMX execution.
12298	*
12299	* @returns Strict VBox status code (i.e. informational status codes too).
12300	* @param pVCpu The cross context virtual CPU structure.
12301	* @param pVmxTransient The VMX-transient structure.
12302	*/
12303	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12304	{
12305	#ifdef DEBUG_ramshankar
12306	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12307	do { \
12308	if (a_fSave != 0) \
12309	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
12310	VBOXSTRICTRC rcStrict = a_CallExpr; \
12311	if (a_fSave != 0) \
12312	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12313	return rcStrict; \
12314	} while (0)
12315	#else
12316	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12317	#endif
12318	uint32_t const rcReason = pVmxTransient->uExitReason;
12319	switch (rcReason)
12320	{
12321	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12322	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12323	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12324	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12325	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12326	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12327	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12328	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12329	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12330	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12331	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12332	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12333	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12334	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12335	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12336	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12337	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12338	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12339	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12340	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12341	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12342	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12343	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12344	case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
12345	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12346	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12347	case VMX_EXIT_GDTR_IDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12348	case VMX_EXIT_LDTR_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12349	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12350	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12351	case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
12352	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12353	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12354	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12355	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12356	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12357	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12358	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12359	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12360	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12361	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12362	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12363	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12364	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12365	#else
12366	case VMX_EXIT_VMCLEAR:
12367	case VMX_EXIT_VMLAUNCH:
12368	case VMX_EXIT_VMPTRLD:
12369	case VMX_EXIT_VMPTRST:
12370	case VMX_EXIT_VMREAD:
12371	case VMX_EXIT_VMRESUME:
12372	case VMX_EXIT_VMWRITE:
12373	case VMX_EXIT_VMXOFF:
12374	case VMX_EXIT_VMXON:
12375	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12376	#endif
12377
12378	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12379	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12380	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
12381	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
12382	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
12383	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
12384	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
12385	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12386	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
12387
12388	case VMX_EXIT_INVEPT:
12389	case VMX_EXIT_INVVPID:
12390	case VMX_EXIT_VMFUNC:
12391	case VMX_EXIT_XSAVES:
12392	case VMX_EXIT_XRSTORS:
12393	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12394
12395	case VMX_EXIT_ENCLS:
12396	case VMX_EXIT_RDSEED:
12397	case VMX_EXIT_PML_FULL:
12398	default:
12399	return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
12400	}
12401	#undef VMEXIT_CALL_RET
12402	}
12403	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12404
12405
12406	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12407	/**
12408	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12409	*
12410	* @returns Strict VBox status code (i.e. informational status codes too).
12411	* @param pVCpu The cross context virtual CPU structure.
12412	* @param pVmxTransient The VMX-transient structure.
12413	*/
12414	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12415	{
12416	uint32_t const rcReason = pVmxTransient->uExitReason;
12417	switch (rcReason)
12418	{
12419	case VMX_EXIT_EPT_MISCONFIG:
12420	case VMX_EXIT_EPT_VIOLATION:
12421	case VMX_EXIT_IO_INSTR:
12422	case VMX_EXIT_CPUID:
12423	case VMX_EXIT_RDTSC:
12424	case VMX_EXIT_RDTSCP:
12425	case VMX_EXIT_APIC_ACCESS:
12426	case VMX_EXIT_XCPT_OR_NMI:
12427	case VMX_EXIT_MOV_CRX:
12428	case VMX_EXIT_EXT_INT:
12429	case VMX_EXIT_INT_WINDOW:
12430	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12431	case VMX_EXIT_MWAIT:
12432	case VMX_EXIT_MONITOR:
12433	case VMX_EXIT_TASK_SWITCH:
12434	case VMX_EXIT_PREEMPT_TIMER:
12435	case VMX_EXIT_RDMSR:
12436	case VMX_EXIT_WRMSR:
12437	case VMX_EXIT_VMCALL:
12438	case VMX_EXIT_MOV_DRX:
12439	case VMX_EXIT_HLT:
12440	case VMX_EXIT_INVD:
12441	case VMX_EXIT_INVLPG:
12442	case VMX_EXIT_RSM:
12443	case VMX_EXIT_MTF:
12444	case VMX_EXIT_PAUSE:
12445	case VMX_EXIT_GDTR_IDTR_ACCESS:
12446	case VMX_EXIT_LDTR_TR_ACCESS:
12447	case VMX_EXIT_WBINVD:
12448	case VMX_EXIT_XSETBV:
12449	case VMX_EXIT_RDRAND:
12450	case VMX_EXIT_INVPCID:
12451	case VMX_EXIT_GETSEC:
12452	case VMX_EXIT_RDPMC:
12453	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12454	case VMX_EXIT_VMCLEAR:
12455	case VMX_EXIT_VMLAUNCH:
12456	case VMX_EXIT_VMPTRLD:
12457	case VMX_EXIT_VMPTRST:
12458	case VMX_EXIT_VMREAD:
12459	case VMX_EXIT_VMRESUME:
12460	case VMX_EXIT_VMWRITE:
12461	case VMX_EXIT_VMXOFF:
12462	case VMX_EXIT_VMXON:
12463	#endif
12464	case VMX_EXIT_TRIPLE_FAULT:
12465	case VMX_EXIT_NMI_WINDOW:
12466	case VMX_EXIT_INIT_SIGNAL:
12467	case VMX_EXIT_SIPI:
12468	case VMX_EXIT_IO_SMI:
12469	case VMX_EXIT_SMI:
12470	case VMX_EXIT_ERR_MSR_LOAD:
12471	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12472	case VMX_EXIT_ERR_MACHINE_CHECK:
12473
12474	case VMX_EXIT_INVEPT:
12475	case VMX_EXIT_INVVPID:
12476	case VMX_EXIT_VMFUNC:
12477	case VMX_EXIT_XSAVES:
12478	case VMX_EXIT_XRSTORS:
12479
12480	case VMX_EXIT_ENCLS:
12481	case VMX_EXIT_RDSEED:
12482	case VMX_EXIT_PML_FULL:
12483	default:
12484	return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
12485	}
12486	#undef VMEXIT_CALL_RET
12487	}
12488	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12489
12490
12491	#ifdef VBOX_STRICT
12492	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
12493	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
12494	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
12495
12496	# define HMVMX_ASSERT_PREEMPT_CPUID() \
12497	do { \
12498	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
12499	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
12500	} while (0)
12501
12502	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12503	do { \
12504	AssertPtr((a_pVCpu)); \
12505	AssertPtr((a_pVmxTransient)); \
12506	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
12507	Assert((a_pVmxTransient)->pVmcsInfo); \
12508	Assert(ASMIntAreEnabled()); \
12509	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12510	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
12511	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
12512	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12513	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
12514	HMVMX_ASSERT_PREEMPT_CPUID(); \
12515	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12516	} while (0)
12517
12518	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12519	do { \
12520	Log4Func(("\n")); \
12521	} while (0)
12522	#else
12523	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12524	do { \
12525	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12526	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
12527	} while (0)
12528	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
12529	#endif
12530
12531
12532	/**
12533	* Advances the guest RIP by the specified number of bytes.
12534	*
12535	* @param pVCpu The cross context virtual CPU structure.
12536	* @param cbInstr Number of bytes to advance the RIP by.
12537	*
12538	* @remarks No-long-jump zone!!!
12539	*/
12540	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
12541	{
12542	/* Advance the RIP. */
12543	pVCpu->cpum.GstCtx.rip += cbInstr;
12544	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12545
12546	/* Update interrupt inhibition. */
12547	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
12548	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
12549	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12550	}
12551
12552
12553	/**
12554	* Advances the guest RIP after reading it from the VMCS.
12555	*
12556	* @returns VBox status code, no informational status codes.
12557	* @param pVCpu The cross context virtual CPU structure.
12558	* @param pVmxTransient The VMX-transient structure.
12559	*
12560	* @remarks No-long-jump zone!!!
12561	*/
12562	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12563	{
12564	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12565	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
12566	AssertRCReturn(rc, rc);
12567
12568	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
12569	return VINF_SUCCESS;
12570	}
12571
12572
12573	/**
12574	* Handle a condition that occurred while delivering an event through the guest
12575	* IDT.
12576	*
12577	* @returns Strict VBox status code (i.e. informational status codes too).
12578	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
12579	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
12580	* to continue execution of the guest which will delivery the \#DF.
12581	* @retval VINF_EM_RESET if we detected a triple-fault condition.
12582	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
12583	*
12584	* @param pVCpu The cross context virtual CPU structure.
12585	* @param pVmxTransient The VMX-transient structure.
12586	*
12587	* @remarks No-long-jump zone!!!
12588	*/
12589	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12590	{
12591	uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12592
12593	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12594	rc2 \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12595	AssertRCReturn(rc2, rc2);
12596
12597	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
12598	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12599	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12600	{
12601	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12602	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12603
12604	/*
12605	* If the event was a software interrupt (generated with INT n) or a software exception
12606	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
12607	* can handle the VM-exit and continue guest execution which will re-execute the
12608	* instruction rather than re-injecting the exception, as that can cause premature
12609	* trips to ring-3 before injection and involve TRPM which currently has no way of
12610	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
12611	* the problem).
12612	*/
12613	IEMXCPTRAISE enmRaise;
12614	IEMXCPTRAISEINFO fRaiseInfo;
12615	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
12616	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
12617	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
12618	{
12619	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
12620	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12621	}
12622	else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
12623	{
12624	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12625	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
12626	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
12627	/** @todo Make AssertMsgReturn as just AssertMsg later. */
12628	AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
12629	("Unexpected VM-exit interruption vector type %#x!\n", uExitVectorType), VERR_VMX_IPE_5);
12630
12631	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
12632
12633	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
12634	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
12635	{
12636	pVmxTransient->fVectoringPF = true;
12637	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12638	}
12639	}
12640	else
12641	{
12642	/*
12643	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
12644	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
12645	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
12646	*/
12647	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12648	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12649	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
12650	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12651	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12652	}
12653
12654	/*
12655	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
12656	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
12657	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
12658	* subsequent VM-entry would fail.
12659	*
12660	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
12661	*/
12662	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS)
12663	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12664	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
12665	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
12666	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
12667	{
12668	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
12669	}
12670
12671	switch (enmRaise)
12672	{
12673	case IEMXCPTRAISE_CURRENT_XCPT:
12674	{
12675	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
12676	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
12677	Assert(rcStrict == VINF_SUCCESS);
12678	break;
12679	}
12680
12681	case IEMXCPTRAISE_PREV_EVENT:
12682	{
12683	uint32_t u32ErrCode;
12684	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
12685	{
12686	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12687	AssertRCReturn(rc2, rc2);
12688	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12689	}
12690	else
12691	u32ErrCode = 0;
12692
12693	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
12694	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
12695	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12696	0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
12697
12698	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
12699	pVCpu->hm.s.Event.u32ErrCode));
12700	Assert(rcStrict == VINF_SUCCESS);
12701	break;
12702	}
12703
12704	case IEMXCPTRAISE_REEXEC_INSTR:
12705	Assert(rcStrict == VINF_SUCCESS);
12706	break;
12707
12708	case IEMXCPTRAISE_DOUBLE_FAULT:
12709	{
12710	/*
12711	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
12712	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
12713	*/
12714	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
12715	{
12716	pVmxTransient->fVectoringDoublePF = true;
12717	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
12718	pVCpu->cpum.GstCtx.cr2));
12719	rcStrict = VINF_SUCCESS;
12720	}
12721	else
12722	{
12723	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
12724	hmR0VmxSetPendingXcptDF(pVCpu);
12725	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
12726	uIdtVector, uExitVector));
12727	rcStrict = VINF_HM_DOUBLE_FAULT;
12728	}
12729	break;
12730	}
12731
12732	case IEMXCPTRAISE_TRIPLE_FAULT:
12733	{
12734	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
12735	rcStrict = VINF_EM_RESET;
12736	break;
12737	}
12738
12739	case IEMXCPTRAISE_CPU_HANG:
12740	{
12741	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
12742	rcStrict = VERR_EM_GUEST_CPU_HANG;
12743	break;
12744	}
12745
12746	default:
12747	{
12748	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
12749	rcStrict = VERR_VMX_IPE_2;
12750	break;
12751	}
12752	}
12753	}
12754	else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
12755	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
12756	&& uExitVector != X86_XCPT_DF
12757	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
12758	{
12759	/*
12760	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
12761	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
12762	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
12763	*/
12764	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
12765	{
12766	Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
12767	VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
12768	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
12769	}
12770	}
12771
12772	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
12773	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
12774	return rcStrict;
12775	}
12776
12777
12778	/** @name VM-exit handlers.
12779	* @{
12780	*/
12781	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12782	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12783	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12784
12785	/**
12786	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
12787	*/
12788	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12789	{
12790	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12791	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
12792	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
12793	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
12794	return VINF_SUCCESS;
12795	return VINF_EM_RAW_INTERRUPT;
12796	}
12797
12798
12799	/**
12800	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
12801	*/
12802	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12803	{
12804	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12805	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
12806
12807	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12808	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12809	AssertRCReturn(rc, rc);
12810
12811	uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12812	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
12813	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
12814	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
12815
12816	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12817	{
12818	/*
12819	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
12820	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
12821	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
12822	*
12823	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
12824	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
12825	*/
12826	VMXDispatchHostNmi();
12827	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
12828	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12829	return VINF_SUCCESS;
12830	}
12831
12832	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12833	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12834	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
12835	{ /* likely */ }
12836	else
12837	{
12838	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
12839	rcStrictRc1 = VINF_SUCCESS;
12840	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12841	return rcStrictRc1;
12842	}
12843
12844	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
12845	uint32_t uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
12846	switch (uIntType)
12847	{
12848	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
12849	Assert(uVector == X86_XCPT_DB);
12850	RT_FALL_THRU();
12851	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
12852	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
12853	RT_FALL_THRU();
12854	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
12855	{
12856	/*
12857	* If there's any exception caused as a result of event injection, the resulting
12858	* secondary/final execption will be pending, we shall continue guest execution
12859	* after injecting the event. The page-fault case is complicated and we manually
12860	* handle any currently pending event in hmR0VmxExitXcptPF.
12861	*/
12862	if (!pVCpu->hm.s.Event.fPending)
12863	{ /* likely */ }
12864	else if (uVector != X86_XCPT_PF)
12865	{
12866	rc = VINF_SUCCESS;
12867	break;
12868	}
12869
12870	switch (uVector)
12871	{
12872	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
12873	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
12874	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
12875	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
12876	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
12877	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
12878
12879	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
12880	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12881	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
12882	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12883	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
12884	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12885	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
12886	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12887	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
12888	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12889	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
12890	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12891	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
12892	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12893	default:
12894	{
12895	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
12896	if (pVmcsInfo->RealMode.fRealOnV86Active)
12897	{
12898	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
12899	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
12900	Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
12901
12902	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
12903	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12904	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12905	AssertRCReturn(rc, rc);
12906	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
12907	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
12908	0 /* GCPtrFaultAddress */);
12909	}
12910	else
12911	{
12912	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
12913	pVCpu->hm.s.u32HMError = uVector;
12914	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
12915	}
12916	break;
12917	}
12918	}
12919	break;
12920	}
12921
12922	default:
12923	{
12924	pVCpu->hm.s.u32HMError = uExitIntInfo;
12925	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
12926	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
12927	break;
12928	}
12929	}
12930	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12931	return rc;
12932	}
12933
12934
12935	/**
12936	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
12937	*/
12938	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12939	{
12940	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12941
12942	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
12943	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12944	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
12945	AssertRCReturn(rc, rc);
12946
12947	/* Evaluate and deliver pending events and resume guest execution. */
12948	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
12949	return VINF_SUCCESS;
12950	}
12951
12952
12953	/**
12954	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
12955	*/
12956	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12957	{
12958	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12959
12960	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12961	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
12962	{
12963	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
12964	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12965	}
12966
12967	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS));
12968
12969	/*
12970	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
12971	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
12972	*/
12973	uint32_t fIntrState;
12974	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
12975	AssertRCReturn(rc, rc);
12976	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
12977	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
12978	{
12979	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
12980	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12981
12982	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
12983	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
12984	AssertRCReturn(rc, rc);
12985	}
12986
12987	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
12988	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
12989	AssertRCReturn(rc, rc);
12990
12991	/* Evaluate and deliver pending events and resume guest execution. */
12992	return VINF_SUCCESS;
12993	}
12994
12995
12996	/**
12997	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
12998	*/
12999	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13000	{
13001	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13002	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13003	}
13004
13005
13006	/**
13007	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
13008	*/
13009	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13010	{
13011	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13012	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13013	}
13014
13015
13016	/**
13017	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
13018	*/
13019	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13020	{
13021	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13022
13023	/*
13024	* Get the state we need and update the exit history entry.
13025	*/
13026	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13027	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13028	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13029	AssertRCReturn(rc, rc);
13030
13031	VBOXSTRICTRC rcStrict;
13032	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
13033	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
13034	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
13035	if (!pExitRec)
13036	{
13037	/*
13038	* Regular CPUID instruction execution.
13039	*/
13040	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
13041	if (rcStrict == VINF_SUCCESS)
13042	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13043	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13044	{
13045	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13046	rcStrict = VINF_SUCCESS;
13047	}
13048	}
13049	else
13050	{
13051	/*
13052	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
13053	*/
13054	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13055	AssertRCReturn(rc2, rc2);
13056
13057	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
13058	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
13059
13060	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
13061	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13062
13063	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
13064	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13065	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
13066	}
13067	return rcStrict;
13068	}
13069
13070
13071	/**
13072	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
13073	*/
13074	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13075	{
13076	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13077
13078	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13079	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
13080	AssertRCReturn(rc, rc);
13081
13082	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
13083	return VINF_EM_RAW_EMULATE_INSTR;
13084
13085	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
13086	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13087	}
13088
13089
13090	/**
13091	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
13092	*/
13093	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13094	{
13095	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13096
13097	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13098	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13099	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13100	AssertRCReturn(rc, rc);
13101
13102	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
13103	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13104	{
13105	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13106	we must reset offsetting on VM-entry. See @bugref{6634}. */
13107	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13108	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13109	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13110	}
13111	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13112	{
13113	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13114	rcStrict = VINF_SUCCESS;
13115	}
13116	return rcStrict;
13117	}
13118
13119
13120	/**
13121	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
13122	*/
13123	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13124	{
13125	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13126
13127	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13128	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
13129	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13130	AssertRCReturn(rc, rc);
13131
13132	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
13133	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13134	{
13135	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13136	we must reset offsetting on VM-reentry. See @bugref{6634}. */
13137	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13138	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13139	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13140	}
13141	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13142	{
13143	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13144	rcStrict = VINF_SUCCESS;
13145	}
13146	return rcStrict;
13147	}
13148
13149
13150	/**
13151	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
13152	*/
13153	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13154	{
13155	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13156
13157	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13158	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
13159	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13160	AssertRCReturn(rc, rc);
13161
13162	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13163	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13164	if (RT_LIKELY(rc == VINF_SUCCESS))
13165	{
13166	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13167	Assert(pVmxTransient->cbInstr == 2);
13168	}
13169	else
13170	{
13171	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
13172	rc = VERR_EM_INTERPRETER;
13173	}
13174	return rc;
13175	}
13176
13177
13178	/**
13179	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
13180	*/
13181	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13182	{
13183	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13184
13185	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
13186	if (EMAreHypercallInstructionsEnabled(pVCpu))
13187	{
13188	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13189	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
13190	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13191	AssertRCReturn(rc, rc);
13192
13193	/* Perform the hypercall. */
13194	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
13195	if (rcStrict == VINF_SUCCESS)
13196	{
13197	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13198	AssertRCReturn(rc, rc);
13199	}
13200	else
13201	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
13202	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
13203	\|\| RT_FAILURE(rcStrict));
13204
13205	/* If the hypercall changes anything other than guest's general-purpose registers,
13206	we would need to reload the guest changed bits here before VM-entry. */
13207	}
13208	else
13209	Log4Func(("Hypercalls not enabled\n"));
13210
13211	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
13212	if (RT_FAILURE(rcStrict))
13213	{
13214	hmR0VmxSetPendingXcptUD(pVCpu);
13215	rcStrict = VINF_SUCCESS;
13216	}
13217
13218	return rcStrict;
13219	}
13220
13221
13222	/**
13223	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
13224	*/
13225	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13226	{
13227	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13228	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
13229
13230	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13231	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13232	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13233	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13234	AssertRCReturn(rc, rc);
13235
13236	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
13237
13238	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
13239	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13240	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13241	{
13242	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13243	rcStrict = VINF_SUCCESS;
13244	}
13245	else
13246	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n", pVmxTransient->uExitQual,
13247	VBOXSTRICTRC_VAL(rcStrict)));
13248	return rcStrict;
13249	}
13250
13251
13252	/**
13253	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
13254	*/
13255	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13256	{
13257	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13258
13259	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13260	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13261	AssertRCReturn(rc, rc);
13262
13263	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13264	rc = EMInterpretMonitor(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13265	if (RT_LIKELY(rc == VINF_SUCCESS))
13266	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13267	else
13268	{
13269	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
13270	rc = VERR_EM_INTERPRETER;
13271	}
13272	return rc;
13273	}
13274
13275
13276	/**
13277	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
13278	*/
13279	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13280	{
13281	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13282
13283	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13284	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13285	AssertRCReturn(rc, rc);
13286
13287	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13288	VBOXSTRICTRC rc2 = EMInterpretMWait(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13289	rc = VBOXSTRICTRC_VAL(rc2);
13290	if (RT_LIKELY( rc == VINF_SUCCESS
13291	\|\| rc == VINF_EM_HALT))
13292	{
13293	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13294	AssertRCReturn(rc3, rc3);
13295
13296	if ( rc == VINF_EM_HALT
13297	&& EMMonitorWaitShouldContinue(pVCpu, pCtx))
13298	rc = VINF_SUCCESS;
13299	}
13300	else
13301	{
13302	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
13303	rc = VERR_EM_INTERPRETER;
13304	}
13305	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
13306	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
13307	return rc;
13308	}
13309
13310
13311	/**
13312	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
13313	*/
13314	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13315	{
13316	/*
13317	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
13318	* mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
13319	* treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
13320	* VMX root operation. If we get here, something funny is going on.
13321	*
13322	* See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
13323	*/
13324	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13325	AssertMsgFailed(("Unexpected RSM VM-exit\n"));
13326	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13327	}
13328
13329
13330	/**
13331	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
13332	*/
13333	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13334	{
13335	/*
13336	* This can only happen if we support dual-monitor treatment of SMI, which can be activated
13337	* by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
13338	* this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
13339	* an SMI. If we get here, something funny is going on.
13340	*
13341	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
13342	* See Intel spec. 25.3 "Other Causes of VM-Exits"
13343	*/
13344	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13345	AssertMsgFailed(("Unexpected SMI VM-exit\n"));
13346	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13347	}
13348
13349
13350	/**
13351	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
13352	*/
13353	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13354	{
13355	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
13356	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13357	AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
13358	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13359	}
13360
13361
13362	/**
13363	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
13364	*/
13365	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13366	{
13367	/*
13368	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
13369	* We don't make use of it as our guests don't have direct access to the host LAPIC.
13370	* See Intel spec. 25.3 "Other Causes of VM-exits".
13371	*/
13372	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13373	AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
13374	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13375	}
13376
13377
13378	/**
13379	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
13380	* VM-exit.
13381	*/
13382	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13383	{
13384	/*
13385	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
13386	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
13387	*
13388	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
13389	* See Intel spec. "23.8 Restrictions on VMX operation".
13390	*/
13391	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13392	return VINF_SUCCESS;
13393	}
13394
13395
13396	/**
13397	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
13398	* VM-exit.
13399	*/
13400	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13401	{
13402	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13403	return VINF_EM_RESET;
13404	}
13405
13406
13407	/**
13408	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
13409	*/
13410	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13411	{
13412	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13413
13414	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13415	AssertRCReturn(rc, rc);
13416
13417	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
13418	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
13419	rc = VINF_SUCCESS;
13420	else
13421	rc = VINF_EM_HALT;
13422
13423	if (rc != VINF_SUCCESS)
13424	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
13425	return rc;
13426	}
13427
13428
13429	/**
13430	* VM-exit handler for instructions that result in a \#UD exception delivered to
13431	* the guest.
13432	*/
13433	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13434	{
13435	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13436	hmR0VmxSetPendingXcptUD(pVCpu);
13437	return VINF_SUCCESS;
13438	}
13439
13440
13441	/**
13442	* VM-exit handler for expiry of the VMX-preemption timer.
13443	*/
13444	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13445	{
13446	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13447
13448	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
13449	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13450
13451	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
13452	PVM pVM = pVCpu->CTX_SUFF(pVM);
13453	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
13454	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
13455	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
13456	}
13457
13458
13459	/**
13460	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
13461	*/
13462	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13463	{
13464	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13465
13466	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13467	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13468	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
13469	AssertRCReturn(rc, rc);
13470
13471	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
13472	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13473	: HM_CHANGED_RAISED_XCPT_MASK);
13474
13475	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13476	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
13477
13478	return rcStrict;
13479	}
13480
13481
13482	/**
13483	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
13484	*/
13485	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13486	{
13487	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13488	/** @todo Use VM-exit instruction information. */
13489	return VERR_EM_INTERPRETER;
13490	}
13491
13492
13493	/**
13494	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
13495	* Error VM-exit.
13496	*/
13497	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13498	{
13499	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13500	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13501	AssertRCReturn(rc, rc);
13502
13503	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo);
13504	if (RT_FAILURE(rc))
13505	return rc;
13506
13507	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
13508	NOREF(uInvalidReason);
13509
13510	#ifdef VBOX_STRICT
13511	uint32_t fIntrState;
13512	RTHCUINTREG uHCReg;
13513	uint64_t u64Val;
13514	uint32_t u32Val;
13515	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
13516	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
13517	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
13518	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13519	AssertRCReturn(rc, rc);
13520
13521	Log4(("uInvalidReason %u\n", uInvalidReason));
13522	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
13523	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
13524	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
13525	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
13526
13527	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
13528	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
13529	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
13530	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
13531	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
13532	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13533	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
13534	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
13535	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
13536	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13537	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
13538	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
13539
13540	hmR0DumpRegs(pVCpu);
13541	#endif
13542
13543	return VERR_VMX_INVALID_GUEST_STATE;
13544	}
13545
13546
13547	/**
13548	* VM-exit handler for VM-entry failure due to an MSR-load
13549	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
13550	*/
13551	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13552	{
13553	AssertMsgFailed(("Unexpected MSR-load exit\n"));
13554	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13555	}
13556
13557
13558	/**
13559	* VM-exit handler for VM-entry failure due to a machine-check event
13560	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
13561	*/
13562	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13563	{
13564	AssertMsgFailed(("Unexpected machine-check event exit\n"));
13565	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13566	}
13567
13568
13569	/**
13570	* VM-exit handler for all undefined reasons. Should never ever happen.. in
13571	* theory.
13572	*/
13573	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13574	{
13575	RT_NOREF2(pVCpu, pVmxTransient);
13576	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
13577	return VERR_VMX_UNDEFINED_EXIT_CODE;
13578	}
13579
13580
13581	/**
13582	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
13583	* (VMX_EXIT_GDTR_IDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
13584	* Conditional VM-exit.
13585	*/
13586	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13587	{
13588	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13589
13590	/* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
13591	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
13592	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13593	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
13594	return VERR_EM_INTERPRETER;
13595	AssertMsgFailed(("Unexpected XDTR access\n"));
13596	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13597	}
13598
13599
13600	/**
13601	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
13602	*/
13603	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13604	{
13605	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13606
13607	/* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
13608	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13609	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
13610	return VERR_EM_INTERPRETER;
13611	AssertMsgFailed(("Unexpected RDRAND exit\n"));
13612	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13613	}
13614
13615
13616	/**
13617	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
13618	*/
13619	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13620	{
13621	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13622
13623	/** @todo Optimize this: We currently drag in in the whole MSR state
13624	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13625	* MSRs required. That would require changes to IEM and possibly CPUM too.
13626	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13627	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13628	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13629	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13630	switch (idMsr)
13631	{
13632	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13633	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13634	}
13635
13636	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13637	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13638	AssertRCReturn(rc, rc);
13639
13640	Log4Func(("ecx=%#RX32\n", idMsr));
13641
13642	#ifdef VBOX_STRICT
13643	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
13644	{
13645	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
13646	&& idMsr != MSR_K6_EFER)
13647	{
13648	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
13649	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13650	}
13651	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13652	{
13653	Assert(pVmcsInfo->pvMsrBitmap);
13654	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13655	if (fMsrpm & VMXMSRPM_ALLOW_RD)
13656	{
13657	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
13658	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13659	}
13660	}
13661	}
13662	#endif
13663
13664	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
13665	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
13666	if (rcStrict == VINF_SUCCESS)
13667	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13668	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
13669	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13670	{
13671	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13672	rcStrict = VINF_SUCCESS;
13673	}
13674	else
13675	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13676
13677	return rcStrict;
13678	}
13679
13680
13681	/**
13682	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
13683	*/
13684	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13685	{
13686	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13687
13688	/** @todo Optimize this: We currently drag in in the whole MSR state
13689	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13690	* MSRs required. That would require changes to IEM and possibly CPUM too.
13691	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13692	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13693	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13694
13695	/*
13696	* The FS and GS base MSRs are not part of the above all-MSRs mask.
13697	* Although we don't need to fetch the base as it will be overwritten shortly, while
13698	* loading guest-state we would also load the entire segment register including limit
13699	* and attributes and thus we need to load them here.
13700	*/
13701	switch (idMsr)
13702	{
13703	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13704	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13705	}
13706
13707	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13708	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13709	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13710	AssertRCReturn(rc, rc);
13711
13712	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
13713
13714	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
13715	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
13716
13717	if (rcStrict == VINF_SUCCESS)
13718	{
13719	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13720
13721	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
13722	if ( idMsr == MSR_IA32_APICBASE
13723	\|\| ( idMsr >= MSR_IA32_X2APIC_START
13724	&& idMsr <= MSR_IA32_X2APIC_END))
13725	{
13726	/*
13727	* We've already saved the APIC related guest-state (TPR) in post-run phase.
13728	* When full APIC register virtualization is implemented we'll have to make
13729	* sure APIC state is saved from the VMCS before IEM changes it.
13730	*/
13731	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
13732	}
13733	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
13734	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13735	else if (idMsr == MSR_K6_EFER)
13736	{
13737	/*
13738	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
13739	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
13740	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
13741	*/
13742	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
13743	}
13744
13745	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
13746	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
13747	{
13748	switch (idMsr)
13749	{
13750	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
13751	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
13752	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
13753	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
13754	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
13755	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
13756	default:
13757	{
13758	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
13759	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
13760	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13761	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
13762	break;
13763	}
13764	}
13765	}
13766	#ifdef VBOX_STRICT
13767	else
13768	{
13769	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
13770	switch (idMsr)
13771	{
13772	case MSR_IA32_SYSENTER_CS:
13773	case MSR_IA32_SYSENTER_EIP:
13774	case MSR_IA32_SYSENTER_ESP:
13775	case MSR_K8_FS_BASE:
13776	case MSR_K8_GS_BASE:
13777	{
13778	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13779	Assert(fMsrpm == (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR));
13780
13781	uint32_t u32Proc;
13782	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Proc);
13783	AssertRC(rc);
13784	Assert(u32Proc == pVmcsInfo->u32ProcCtls);
13785	Assert(u32Proc & VMX_PROC_CTLS_USE_MSR_BITMAPS);
13786
13787	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32 fMsrpm=%#RX32\n", idMsr, fMsrpm));
13788	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13789	}
13790
13791	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
13792	default:
13793	{
13794	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
13795	{
13796	/* EFER MSR writes are always intercepted. */
13797	if (idMsr != MSR_K6_EFER)
13798	{
13799	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
13800	idMsr));
13801	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13802	}
13803	}
13804
13805	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13806	{
13807	Assert(pVmcsInfo->pvMsrBitmap);
13808	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13809	if (fMsrpm & VMXMSRPM_ALLOW_WR)
13810	{
13811	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
13812	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13813	}
13814	}
13815	break;
13816	}
13817	}
13818	}
13819	#endif /* VBOX_STRICT */
13820	}
13821	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13822	{
13823	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13824	rcStrict = VINF_SUCCESS;
13825	}
13826	else
13827	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13828
13829	return rcStrict;
13830	}
13831
13832
13833	/**
13834	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
13835	*/
13836	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13837	{
13838	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13839	/** @todo The guest has likely hit a contended spinlock. We might want to
13840	* poke a schedule different guest VCPU. */
13841	return VINF_EM_RAW_INTERRUPT;
13842	}
13843
13844
13845	/**
13846	* VM-exit handler for when the TPR value is lowered below the specified
13847	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
13848	*/
13849	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13850	{
13851	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13852	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
13853
13854	/*
13855	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
13856	* We'll re-evaluate pending interrupts and inject them before the next VM
13857	* entry so we can just continue execution here.
13858	*/
13859	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
13860	return VINF_SUCCESS;
13861	}
13862
13863
13864	/**
13865	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
13866	* VM-exit.
13867	*
13868	* @retval VINF_SUCCESS when guest execution can continue.
13869	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
13870	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
13871	* interpreter.
13872	*/
13873	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13874	{
13875	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13876	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
13877
13878	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13879	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13880	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13881	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13882	AssertRCReturn(rc, rc);
13883
13884	VBOXSTRICTRC rcStrict;
13885	PVM pVM = pVCpu->CTX_SUFF(pVM);
13886	RTGCUINTPTR const uExitQual = pVmxTransient->uExitQual;
13887	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
13888	switch (uAccessType)
13889	{
13890	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
13891	{
13892	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
13893	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
13894	VMX_EXIT_QUAL_CRX_GENREG(uExitQual));
13895	AssertMsg( rcStrict == VINF_SUCCESS
13896	\|\| rcStrict == VINF_IEM_RAISED_XCPT
13897	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13898
13899	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
13900	{
13901	case 0:
13902	{
13903	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13904	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13905	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
13906	Log4Func(("CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
13907
13908	/*
13909	* This is a kludge for handling switches back to real mode when we try to use
13910	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
13911	* deal with special selector values, so we have to return to ring-3 and run
13912	* there till the selector values are V86 mode compatible.
13913	*
13914	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
13915	* latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
13916	* at the end of this function.
13917	*/
13918	if ( rc == VINF_SUCCESS
13919	&& !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
13920	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
13921	&& (uOldCr0 & X86_CR0_PE)
13922	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
13923	{
13924	/** @todo check selectors rather than returning all the time. */
13925	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
13926	rcStrict = VINF_EM_RESCHEDULE_REM;
13927	}
13928	break;
13929	}
13930
13931	case 2:
13932	{
13933	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
13934	/* Nothing to do here, CR2 it's not part of the VMCS. */
13935	break;
13936	}
13937
13938	case 3:
13939	{
13940	Assert( !pVM->hm.s.fNestedPaging
13941	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
13942	\|\| pVCpu->hm.s.fUsingDebugLoop);
13943	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
13944	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13945	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
13946	Log4Func(("CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
13947	break;
13948	}
13949
13950	case 4:
13951	{
13952	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
13953	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13954	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
13955	Log4Func(("CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
13956	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
13957	break;
13958	}
13959
13960	case 8:
13961	{
13962	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
13963	Assert(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
13964	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13965	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
13966	break;
13967	}
13968	default:
13969	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual)));
13970	break;
13971	}
13972	break;
13973	}
13974
13975	case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
13976	{
13977	Assert( !pVM->hm.s.fNestedPaging
13978	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
13979	\|\| pVCpu->hm.s.fUsingDebugLoop
13980	\|\| VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 3);
13981	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
13982	Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 8
13983	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
13984
13985	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_GENREG(uExitQual),
13986	VMX_EXIT_QUAL_CRX_REGISTER(uExitQual));
13987	AssertMsg( rcStrict == VINF_SUCCESS
13988	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13989	#ifdef VBOX_WITH_STATISTICS
13990	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
13991	{
13992	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
13993	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
13994	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
13995	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
13996	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
13997	}
13998	#endif
13999	Log4Func(("CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
14000	VBOXSTRICTRC_VAL(rcStrict)));
14001	if (VMX_EXIT_QUAL_CRX_GENREG(uExitQual) == X86_GREG_xSP)
14002	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
14003	else
14004	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14005	break;
14006	}
14007
14008	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
14009	{
14010	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
14011	AssertMsg( rcStrict == VINF_SUCCESS
14012	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14013
14014	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14015	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
14016	Log4Func(("CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
14017	break;
14018	}
14019
14020	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
14021	{
14022	/* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
14023	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
14024	AssertRCReturn(rc, rc);
14025	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual),
14026	pVmxTransient->uGuestLinearAddr);
14027	AssertMsg( rcStrict == VINF_SUCCESS
14028	\|\| rcStrict == VINF_IEM_RAISED_XCPT
14029	, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14030
14031	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14032	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
14033	Log4Func(("LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
14034	break;
14035	}
14036
14037	default:
14038	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
14039	VERR_VMX_UNEXPECTED_EXCEPTION);
14040	}
14041
14042	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
14043	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
14044	if (rcStrict == VINF_IEM_RAISED_XCPT)
14045	{
14046	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14047	rcStrict = VINF_SUCCESS;
14048	}
14049
14050	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
14051	NOREF(pVM);
14052	return rcStrict;
14053	}
14054
14055
14056	/**
14057	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
14058	* VM-exit.
14059	*/
14060	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14061	{
14062	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14063	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
14064
14065	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14066	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14067	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14068	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14069	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
14070	\| CPUMCTX_EXTRN_EFER);
14071	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
14072	AssertRCReturn(rc, rc);
14073
14074	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
14075	uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
14076	uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQual);
14077	bool fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
14078	bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
14079	bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
14080	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
14081	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
14082
14083	/*
14084	* Update exit history to see if this exit can be optimized.
14085	*/
14086	VBOXSTRICTRC rcStrict;
14087	PCEMEXITREC pExitRec = NULL;
14088	if ( !fGstStepping
14089	&& !fDbgStepping)
14090	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14091	!fIOString
14092	? !fIOWrite
14093	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
14094	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
14095	: !fIOWrite
14096	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
14097	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
14098	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14099	if (!pExitRec)
14100	{
14101	/* I/O operation lookup arrays. */
14102	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
14103	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
14104	uint32_t const cbValue = s_aIOSizes[uIOWidth];
14105	uint32_t const cbInstr = pVmxTransient->cbInstr;
14106	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
14107	PVM pVM = pVCpu->CTX_SUFF(pVM);
14108	if (fIOString)
14109	{
14110	/*
14111	* INS/OUTS - I/O String instruction.
14112	*
14113	* Use instruction-information if available, otherwise fall back on
14114	* interpreting the instruction.
14115	*/
14116	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14117	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
14118	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
14119	if (fInsOutsInfo)
14120	{
14121	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14122	AssertRCReturn(rc2, rc2);
14123	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
14124	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
14125	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
14126	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
14127	if (fIOWrite)
14128	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
14129	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
14130	else
14131	{
14132	/*
14133	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
14134	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
14135	* See Intel Instruction spec. for "INS".
14136	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
14137	*/
14138	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
14139	}
14140	}
14141	else
14142	rcStrict = IEMExecOne(pVCpu);
14143
14144	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14145	fUpdateRipAlready = true;
14146	}
14147	else
14148	{
14149	/*
14150	* IN/OUT - I/O instruction.
14151	*/
14152	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14153	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
14154	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
14155	if (fIOWrite)
14156	{
14157	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
14158	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
14159	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14160	&& !pCtx->eflags.Bits.u1TF)
14161	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
14162	}
14163	else
14164	{
14165	uint32_t u32Result = 0;
14166	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
14167	if (IOM_SUCCESS(rcStrict))
14168	{
14169	/* Save result of I/O IN instr. in AL/AX/EAX. */
14170	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
14171	}
14172	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14173	&& !pCtx->eflags.Bits.u1TF)
14174	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
14175	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
14176	}
14177	}
14178
14179	if (IOM_SUCCESS(rcStrict))
14180	{
14181	if (!fUpdateRipAlready)
14182	{
14183	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
14184	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14185	}
14186
14187	/*
14188	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
14189	* while booting Fedora 17 64-bit guest.
14190	*
14191	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
14192	*/
14193	if (fIOString)
14194	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
14195
14196	/*
14197	* If any I/O breakpoints are armed, we need to check if one triggered
14198	* and take appropriate action.
14199	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
14200	*/
14201	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
14202	AssertRCReturn(rc, rc);
14203
14204	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
14205	* execution engines about whether hyper BPs and such are pending. */
14206	uint32_t const uDr7 = pCtx->dr[7];
14207	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
14208	&& X86_DR7_ANY_RW_IO(uDr7)
14209	&& (pCtx->cr4 & X86_CR4_DE))
14210	\|\| DBGFBpIsHwIoArmed(pVM)))
14211	{
14212	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
14213
14214	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
14215	VMMRZCallRing3Disable(pVCpu);
14216	HM_DISABLE_PREEMPT(pVCpu);
14217
14218	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
14219
14220	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
14221	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
14222	{
14223	/* Raise #DB. */
14224	if (fIsGuestDbgActive)
14225	ASMSetDR6(pCtx->dr[6]);
14226	if (pCtx->dr[7] != uDr7)
14227	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
14228
14229	hmR0VmxSetPendingXcptDB(pVCpu);
14230	}
14231	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
14232	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
14233	else if ( rcStrict2 != VINF_SUCCESS
14234	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
14235	rcStrict = rcStrict2;
14236	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
14237
14238	HM_RESTORE_PREEMPT();
14239	VMMRZCallRing3Enable(pVCpu);
14240	}
14241	}
14242
14243	#ifdef VBOX_STRICT
14244	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14245	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
14246	Assert(!fIOWrite);
14247	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14248	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
14249	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
14250	Assert(fIOWrite);
14251	else
14252	{
14253	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
14254	* statuses, that the VMM device and some others may return. See
14255	* IOM_SUCCESS() for guidance. */
14256	AssertMsg( RT_FAILURE(rcStrict)
14257	\|\| rcStrict == VINF_SUCCESS
14258	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
14259	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
14260	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
14261	\|\| rcStrict == VINF_EM_RAW_TO_R3
14262	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14263	# endif
14264	}
14265	#endif
14266	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
14267	}
14268	else
14269	{
14270	/*
14271	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14272	*/
14273	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14274	AssertRCReturn(rc2, rc2);
14275	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
14276	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
14277	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
14278	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14279	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
14280	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
14281
14282	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14283	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14284
14285	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14286	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14287	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14288	}
14289	return rcStrict;
14290	}
14291
14292
14293	/**
14294	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
14295	* VM-exit.
14296	*/
14297	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14298	{
14299	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14300
14301	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
14302	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14303	AssertRCReturn(rc, rc);
14304	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
14305	{
14306	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14307	AssertRCReturn(rc, rc);
14308	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
14309	{
14310	uint32_t uErrCode;
14311	RTGCUINTPTR GCPtrFaultAddress;
14312	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
14313	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
14314	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
14315	if (fErrorCodeValid)
14316	{
14317	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14318	AssertRCReturn(rc, rc);
14319	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
14320	}
14321	else
14322	uErrCode = 0;
14323
14324	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
14325	&& uVector == X86_XCPT_PF)
14326	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
14327	else
14328	GCPtrFaultAddress = 0;
14329
14330	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14331	AssertRCReturn(rc, rc);
14332
14333	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
14334	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
14335
14336	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
14337	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14338	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14339	}
14340	}
14341
14342	/* Fall back to the interpreter to emulate the task-switch. */
14343	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14344	return VERR_EM_INTERPRETER;
14345	}
14346
14347
14348	/**
14349	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
14350	*/
14351	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14352	{
14353	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14354
14355	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14356	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
14357	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14358	AssertRCReturn(rc, rc);
14359	return VINF_EM_DBG_STEPPED;
14360	}
14361
14362
14363	/**
14364	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
14365	*/
14366	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14367	{
14368	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14369	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
14370
14371	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14372	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14373	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14374	{
14375	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
14376	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14377	{
14378	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14379	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14380	}
14381	}
14382	else
14383	{
14384	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14385	rcStrict1 = VINF_SUCCESS;
14386	return rcStrict1;
14387	}
14388
14389	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
14390	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14391	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14392	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14393	AssertRCReturn(rc, rc);
14394
14395	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
14396	uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
14397	VBOXSTRICTRC rcStrict2;
14398	switch (uAccessType)
14399	{
14400	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
14401	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
14402	{
14403	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
14404	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
14405	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
14406
14407	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
14408	GCPhys &= PAGE_BASE_GC_MASK;
14409	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
14410	PVM pVM = pVCpu->CTX_SUFF(pVM);
14411	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
14412	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
14413
14414	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14415	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
14416	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
14417	CPUMCTX2CORE(pCtx), GCPhys);
14418	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14419	if ( rcStrict2 == VINF_SUCCESS
14420	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14421	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14422	{
14423	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14424	\| HM_CHANGED_GUEST_APIC_TPR);
14425	rcStrict2 = VINF_SUCCESS;
14426	}
14427	break;
14428	}
14429
14430	default:
14431	Log4Func(("uAccessType=%#x\n", uAccessType));
14432	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
14433	break;
14434	}
14435
14436	if (rcStrict2 != VINF_SUCCESS)
14437	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
14438	return rcStrict2;
14439	}
14440
14441
14442	/**
14443	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
14444	* VM-exit.
14445	*/
14446	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14447	{
14448	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14449
14450	/* We should -not- get this VM-exit if the guest's debug registers were active. */
14451	if (pVmxTransient->fWasGuestDebugStateActive)
14452	{
14453	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
14454	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14455	}
14456
14457	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14458	if ( !pVCpu->hm.s.fSingleInstruction
14459	&& !pVmxTransient->fWasHyperDebugStateActive)
14460	{
14461	Assert(!DBGFIsStepping(pVCpu));
14462	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
14463
14464	/* Don't intercept MOV DRx any more. */
14465	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
14466	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14467	AssertRCReturn(rc, rc);
14468
14469	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
14470	VMMRZCallRing3Disable(pVCpu);
14471	HM_DISABLE_PREEMPT(pVCpu);
14472
14473	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
14474	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
14475	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
14476
14477	HM_RESTORE_PREEMPT();
14478	VMMRZCallRing3Enable(pVCpu);
14479
14480	#ifdef VBOX_WITH_STATISTICS
14481	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14482	AssertRCReturn(rc, rc);
14483	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14484	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14485	else
14486	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14487	#endif
14488	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
14489	return VINF_SUCCESS;
14490	}
14491
14492	/*
14493	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
14494	* The EFER MSR is always up-to-date.
14495	* Update the segment registers and DR7 from the CPU.
14496	*/
14497	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14498	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14499	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
14500	AssertRCReturn(rc, rc);
14501	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
14502
14503	PVM pVM = pVCpu->CTX_SUFF(pVM);
14504	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14505	{
14506	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14507	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
14508	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
14509	if (RT_SUCCESS(rc))
14510	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
14511	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14512	}
14513	else
14514	{
14515	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14516	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
14517	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
14518	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14519	}
14520
14521	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
14522	if (RT_SUCCESS(rc))
14523	{
14524	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14525	AssertRCReturn(rc2, rc2);
14526	return VINF_SUCCESS;
14527	}
14528	return rc;
14529	}
14530
14531
14532	/**
14533	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
14534	* Conditional VM-exit.
14535	*/
14536	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14537	{
14538	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14539	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14540
14541	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14542	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14543	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14544	{
14545	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
14546	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
14547	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14548	{
14549	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14550	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14551	}
14552	}
14553	else
14554	{
14555	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14556	rcStrict1 = VINF_SUCCESS;
14557	return rcStrict1;
14558	}
14559
14560	/*
14561	* Get sufficent state and update the exit history entry.
14562	*/
14563	RTGCPHYS GCPhys;
14564	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14565	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14566	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14567	AssertRCReturn(rc, rc);
14568
14569	VBOXSTRICTRC rcStrict;
14570	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14571	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
14572	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14573	if (!pExitRec)
14574	{
14575	/*
14576	* If we succeed, resume guest execution.
14577	* If we fail in interpreting the instruction because we couldn't get the guest physical address
14578	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
14579	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
14580	* weird case. See @bugref{6043}.
14581	*/
14582	PVM pVM = pVCpu->CTX_SUFF(pVM);
14583	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14584	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
14585	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
14586	if ( rcStrict == VINF_SUCCESS
14587	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
14588	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
14589	{
14590	/* Successfully handled MMIO operation. */
14591	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14592	\| HM_CHANGED_GUEST_APIC_TPR);
14593	rcStrict = VINF_SUCCESS;
14594	}
14595	}
14596	else
14597	{
14598	/*
14599	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14600	*/
14601	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14602	AssertRCReturn(rc2, rc2);
14603
14604	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
14605	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
14606
14607	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14608	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14609
14610	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14611	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14612	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14613	}
14614	return VBOXSTRICTRC_TODO(rcStrict);
14615	}
14616
14617
14618	/**
14619	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
14620	* VM-exit.
14621	*/
14622	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14623	{
14624	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14625	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14626
14627	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14628	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14629	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14630	{
14631	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
14632	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14633	Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
14634	}
14635	else
14636	{
14637	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14638	rcStrict1 = VINF_SUCCESS;
14639	return rcStrict1;
14640	}
14641
14642	RTGCPHYS GCPhys;
14643	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14644	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14645	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14646	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14647	AssertRCReturn(rc, rc);
14648
14649	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
14650	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
14651
14652	RTGCUINT uErrorCode = 0;
14653	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
14654	uErrorCode \|= X86_TRAP_PF_ID;
14655	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
14656	uErrorCode \|= X86_TRAP_PF_RW;
14657	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
14658	uErrorCode \|= X86_TRAP_PF_P;
14659
14660	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
14661
14662
14663	/* Handle the pagefault trap for the nested shadow table. */
14664	PVM pVM = pVCpu->CTX_SUFF(pVM);
14665	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14666
14667	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
14668	pCtx->cs.Sel, pCtx->rip));
14669
14670	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
14671	TRPMResetTrap(pVCpu);
14672
14673	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
14674	if ( rcStrict2 == VINF_SUCCESS
14675	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14676	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14677	{
14678	/* Successfully synced our nested page tables. */
14679	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
14680	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
14681	return VINF_SUCCESS;
14682	}
14683
14684	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14685	return rcStrict2;
14686	}
14687
14688	/** @} */
14689
14690	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14691	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14692	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14693
14694	/**
14695	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
14696	*/
14697	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14698	{
14699	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14700	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
14701
14702	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
14703	AssertRCReturn(rc, rc);
14704
14705	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
14706	{
14707	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
14708	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
14709
14710	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
14711	* provides VM-exit instruction length. If this causes problem later,
14712	* disassemble the instruction like it's done on AMD-V. */
14713	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14714	AssertRCReturn(rc2, rc2);
14715	return rc;
14716	}
14717
14718	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14719	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14720	return rc;
14721	}
14722
14723
14724	/**
14725	* VM-exit exception handler for \#BP (Breakpoint exception).
14726	*/
14727	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14728	{
14729	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14730	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
14731
14732	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14733	AssertRCReturn(rc, rc);
14734
14735	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14736	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14737	if (rc == VINF_EM_RAW_GUEST_TRAP)
14738	{
14739	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14740	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14741	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14742	AssertRCReturn(rc, rc);
14743
14744	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14745	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14746	}
14747
14748	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
14749	return rc;
14750	}
14751
14752
14753	/**
14754	* VM-exit exception handler for \#AC (alignment check exception).
14755	*/
14756	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14757	{
14758	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14759
14760	/*
14761	* Re-inject it. We'll detect any nesting before getting here.
14762	*/
14763	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14764	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14765	AssertRCReturn(rc, rc);
14766	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
14767
14768	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14769	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14770	return VINF_SUCCESS;
14771	}
14772
14773
14774	/**
14775	* VM-exit exception handler for \#DB (Debug exception).
14776	*/
14777	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14778	{
14779	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14780	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
14781
14782	/*
14783	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
14784	* for processing.
14785	*/
14786	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14787
14788	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
14789	uint64_t uDR6 = X86_DR6_INIT_VAL;
14790	uDR6 \|= (pVmxTransient->uExitQual & (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
14791
14792	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14793	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
14794	Log6Func(("rc=%Rrc\n", rc));
14795	if (rc == VINF_EM_RAW_GUEST_TRAP)
14796	{
14797	/*
14798	* The exception was for the guest. Update DR6, DR7.GD and
14799	* IA32_DEBUGCTL.LBR before forwarding it.
14800	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
14801	*/
14802	VMMRZCallRing3Disable(pVCpu);
14803	HM_DISABLE_PREEMPT(pVCpu);
14804
14805	pCtx->dr[6] &= ~X86_DR6_B_MASK;
14806	pCtx->dr[6] \|= uDR6;
14807	if (CPUMIsGuestDebugStateActive(pVCpu))
14808	ASMSetDR6(pCtx->dr[6]);
14809
14810	HM_RESTORE_PREEMPT();
14811	VMMRZCallRing3Enable(pVCpu);
14812
14813	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
14814	AssertRCReturn(rc, rc);
14815
14816	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
14817	pCtx->dr[7] &= ~X86_DR7_GD;
14818
14819	/* Paranoia. */
14820	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
14821	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
14822
14823	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
14824	AssertRCReturn(rc, rc);
14825
14826	/*
14827	* Raise #DB in the guest.
14828	*
14829	* It is important to reflect exactly what the VM-exit gave us (preserving the
14830	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
14831	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
14832	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
14833	*
14834	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
14835	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
14836	*/
14837	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14838	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14839	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14840	AssertRCReturn(rc, rc);
14841	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14842	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14843	return VINF_SUCCESS;
14844	}
14845
14846	/*
14847	* Not a guest trap, must be a hypervisor related debug event then.
14848	* Update DR6 in case someone is interested in it.
14849	*/
14850	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
14851	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
14852	CPUMSetHyperDR6(pVCpu, uDR6);
14853
14854	return rc;
14855	}
14856
14857
14858	/**
14859	* Hacks its way around the lovely mesa driver's backdoor accesses.
14860	*
14861	* @sa hmR0SvmHandleMesaDrvGp
14862	*/
14863	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14864	{
14865	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
14866	RT_NOREF(pCtx);
14867
14868	/* For now we'll just skip the instruction. */
14869	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14870	}
14871
14872
14873	/**
14874	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
14875	* backdoor logging w/o checking what it is running inside.
14876	*
14877	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
14878	* backdoor port and magic numbers loaded in registers.
14879	*
14880	* @returns true if it is, false if it isn't.
14881	* @sa hmR0SvmIsMesaDrvGp
14882	*/
14883	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14884	{
14885	/* 0xed: IN eAX,dx */
14886	uint8_t abInstr[1];
14887	if (pVmxTransient->cbInstr != sizeof(abInstr))
14888	return false;
14889
14890	/* Check that it is #GP(0). */
14891	if (pVmxTransient->uExitIntErrorCode != 0)
14892	return false;
14893
14894	/* Check magic and port. */
14895	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
14896	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
14897	if (pCtx->rax != UINT32_C(0x564d5868))
14898	return false;
14899	if (pCtx->dx != UINT32_C(0x5658))
14900	return false;
14901
14902	/* Flat ring-3 CS. */
14903	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
14904	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
14905	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
14906	if (pCtx->cs.Attr.n.u2Dpl != 3)
14907	return false;
14908	if (pCtx->cs.u64Base != 0)
14909	return false;
14910
14911	/* Check opcode. */
14912	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
14913	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
14914	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
14915	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
14916	if (RT_FAILURE(rc))
14917	return false;
14918	if (abInstr[0] != 0xed)
14919	return false;
14920
14921	return true;
14922	}
14923
14924
14925	/**
14926	* VM-exit exception handler for \#GP (General-protection exception).
14927	*
14928	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
14929	*/
14930	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14931	{
14932	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14933	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
14934
14935	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14936	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14937	if (pVmcsInfo->RealMode.fRealOnV86Active)
14938	{ /* likely */ }
14939	else
14940	{
14941	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14942	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv);
14943	#endif
14944	/* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
14945	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14946	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14947	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14948	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14949	AssertRCReturn(rc, rc);
14950	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
14951	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
14952
14953	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
14954	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
14955	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14956	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14957	else
14958	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
14959	return rc;
14960	}
14961
14962	Assert(CPUMIsGuestInRealModeEx(pCtx));
14963	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
14964
14965	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14966	AssertRCReturn(rc, rc);
14967
14968	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
14969	if (rcStrict == VINF_SUCCESS)
14970	{
14971	if (!CPUMIsGuestInRealModeEx(pCtx))
14972	{
14973	/*
14974	* The guest is no longer in real-mode, check if we can continue executing the
14975	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
14976	*/
14977	pVmcsInfo->RealMode.fRealOnV86Active = false;
14978	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
14979	{
14980	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
14981	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14982	}
14983	else
14984	{
14985	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
14986	rcStrict = VINF_EM_RESCHEDULE;
14987	}
14988	}
14989	else
14990	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14991	}
14992	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14993	{
14994	rcStrict = VINF_SUCCESS;
14995	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14996	}
14997	return VBOXSTRICTRC_VAL(rcStrict);
14998	}
14999
15000
15001	/**
15002	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
15003	* the exception reported in the VMX transient structure back into the VM.
15004	*
15005	* @remarks Requires uExitIntInfo in the VMX transient structure to be
15006	* up-to-date.
15007	*/
15008	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15009	{
15010	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15011	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
15012	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15013	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active,
15014	("uVector=%#x u32XcptBitmap=%#X32\n",
15015	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
15016	NOREF(pVmcsInfo);
15017	#endif
15018
15019	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
15020	hmR0VmxCheckExitDueToEventDelivery(). */
15021	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15022	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15023	AssertRCReturn(rc, rc);
15024	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15025
15026	#ifdef DEBUG_ramshankar
15027	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
15028	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n",
15029	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pCtx->cs.Sel, pCtx->rip));
15030	#endif
15031
15032	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15033	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15034	return VINF_SUCCESS;
15035	}
15036
15037
15038	/**
15039	* VM-exit exception handler for \#PF (Page-fault exception).
15040	*/
15041	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15042	{
15043	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15044	PVM pVM = pVCpu->CTX_SUFF(pVM);
15045	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15046	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15047	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15048	AssertRCReturn(rc, rc);
15049
15050	if (!pVM->hm.s.fNestedPaging)
15051	{ /* likely */ }
15052	else
15053	{
15054	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
15055	Assert(pVCpu->hm.s.fUsingDebugLoop);
15056	#endif
15057	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
15058	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
15059	{
15060	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15061	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
15062	}
15063	else
15064	{
15065	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15066	hmR0VmxSetPendingXcptDF(pVCpu);
15067	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
15068	}
15069	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15070	return rc;
15071	}
15072
15073	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
15074	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
15075	if (pVmxTransient->fVectoringPF)
15076	{
15077	Assert(pVCpu->hm.s.Event.fPending);
15078	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15079	}
15080
15081	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15082	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15083	AssertRCReturn(rc, rc);
15084
15085	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
15086	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
15087
15088	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
15089	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
15090
15091	Log4Func(("#PF: rc=%Rrc\n", rc));
15092	if (rc == VINF_SUCCESS)
15093	{
15094	/*
15095	* This is typically a shadow page table sync or a MMIO instruction. But we may have
15096	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
15097	*/
15098	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15099	TRPMResetTrap(pVCpu);
15100	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
15101	return rc;
15102	}
15103
15104	if (rc == VINF_EM_RAW_GUEST_TRAP)
15105	{
15106	if (!pVmxTransient->fVectoringDoublePF)
15107	{
15108	/* It's a guest page fault and needs to be reflected to the guest. */
15109	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
15110	TRPMResetTrap(pVCpu);
15111	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
15112	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15113	uGstErrorCode, pVmxTransient->uExitQual);
15114	}
15115	else
15116	{
15117	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15118	TRPMResetTrap(pVCpu);
15119	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
15120	hmR0VmxSetPendingXcptDF(pVCpu);
15121	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
15122	}
15123
15124	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15125	return VINF_SUCCESS;
15126	}
15127
15128	TRPMResetTrap(pVCpu);
15129	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
15130	return rc;
15131	}
15132
15133	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15134	/** @name VMX instruction handlers.
15135	* @{
15136	*/
15137	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15138	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VMX instructions VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15139	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15140
15141	/**
15142	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
15143	*/
15144	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15145	{
15146	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15147
15148	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15149	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15150	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15151	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15152	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15153	AssertRCReturn(rc, rc);
15154
15155	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15156
15157	VMXVEXITINFO ExitInfo;
15158	RT_ZERO(ExitInfo);
15159	ExitInfo.uReason = pVmxTransient->uExitReason;
15160	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15161	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15162	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15163	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15164
15165	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
15166	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15167	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15168	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15169	{
15170	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15171	rcStrict = VINF_SUCCESS;
15172	}
15173	return rcStrict;
15174	}
15175
15176
15177	/**
15178	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
15179	*/
15180	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15181	{
15182	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15183
15184	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
15185	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15186	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15187	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15188	AssertRCReturn(rc, rc);
15189
15190	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15191
15192	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
15193	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15194	{
15195	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15196	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15197	}
15198	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15199	return rcStrict;
15200	}
15201
15202
15203	/**
15204	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
15205	*/
15206	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15207	{
15208	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15209
15210	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15211	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15212	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15213	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15214	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15215	AssertRCReturn(rc, rc);
15216
15217	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15218
15219	VMXVEXITINFO ExitInfo;
15220	RT_ZERO(ExitInfo);
15221	ExitInfo.uReason = pVmxTransient->uExitReason;
15222	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15223	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15224	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15225	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15226
15227	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
15228	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15229	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15230	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15231	{
15232	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15233	rcStrict = VINF_SUCCESS;
15234	}
15235	return rcStrict;
15236	}
15237
15238
15239	/**
15240	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
15241	*/
15242	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15243	{
15244	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15245
15246	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15247	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15248	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15249	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15250	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15251	AssertRCReturn(rc, rc);
15252
15253	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15254
15255	VMXVEXITINFO ExitInfo;
15256	RT_ZERO(ExitInfo);
15257	ExitInfo.uReason = pVmxTransient->uExitReason;
15258	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15259	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15260	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15261	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15262
15263	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
15264	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15265	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15266	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15267	{
15268	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15269	rcStrict = VINF_SUCCESS;
15270	}
15271	return rcStrict;
15272	}
15273
15274
15275	/**
15276	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Unconditional VM-exit.
15277	*/
15278	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15279	{
15280	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15281
15282	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15283	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15284	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15285	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15286	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15287	AssertRCReturn(rc, rc);
15288
15289	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15290
15291	VMXVEXITINFO ExitInfo;
15292	RT_ZERO(ExitInfo);
15293	ExitInfo.uReason = pVmxTransient->uExitReason;
15294	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15295	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15296	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15297	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15298	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15299
15300	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
15301	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15302	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15303	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15304	{
15305	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15306	rcStrict = VINF_SUCCESS;
15307	}
15308	return rcStrict;
15309	}
15310
15311
15312	/**
15313	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
15314	*/
15315	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15316	{
15317	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15318
15319	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
15320	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15321	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15322	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15323	AssertRCReturn(rc, rc);
15324
15325	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15326
15327	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
15328	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15329	{
15330	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15331	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15332	}
15333	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15334	return rcStrict;
15335	}
15336
15337
15338	/**
15339	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Unconditional VM-exit.
15340	*/
15341	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15342	{
15343	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15344
15345	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15346	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15347	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15348	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15349	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15350	AssertRCReturn(rc, rc);
15351
15352	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15353
15354	VMXVEXITINFO ExitInfo;
15355	RT_ZERO(ExitInfo);
15356	ExitInfo.uReason = pVmxTransient->uExitReason;
15357	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15358	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15359	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15360	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15361	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15362
15363	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
15364	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15365	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15366	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15367	{
15368	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15369	rcStrict = VINF_SUCCESS;
15370	}
15371	return rcStrict;
15372	}
15373
15374
15375	/**
15376	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
15377	*/
15378	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15379	{
15380	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15381
15382	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15383	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
15384	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15385	AssertRCReturn(rc, rc);
15386
15387	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15388
15389	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
15390	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15391	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
15392	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15393	{
15394	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15395	rcStrict = VINF_SUCCESS;
15396	}
15397	return rcStrict;
15398	}
15399
15400
15401	/**
15402	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
15403	*/
15404	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15405	{
15406	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15407
15408	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15409	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15410	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15411	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15412	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15413	AssertRCReturn(rc, rc);
15414
15415	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15416
15417	VMXVEXITINFO ExitInfo;
15418	RT_ZERO(ExitInfo);
15419	ExitInfo.uReason = pVmxTransient->uExitReason;
15420	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15421	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15422	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15423	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15424
15425	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
15426	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15427	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15428	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15429	{
15430	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15431	rcStrict = VINF_SUCCESS;
15432	}
15433	return rcStrict;
15434	}
15435
15436	/** @} */
15437	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
15438

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 78231

Download in other formats: