HMVMXR0.cpp@ 78257

Last change on this file since 78257 was 78257, checked in by vboxsync, 6 years ago
VMM/HMVMXR0: Nested VMX: bugref:9180 Assertion.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 641.8 KB

Line
1	/* $Id: HMVMXR0.cpp 78257 2019-04-23 08:19:10Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27
28	#include <VBox/vmm/pdmapi.h>
29	#include <VBox/vmm/dbgf.h>
30	#include <VBox/vmm/iem.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#include <VBox/vmm/em.h>
35	#include <VBox/vmm/gim.h>
36	#include <VBox/vmm/apic.h>
37	#ifdef VBOX_WITH_REM
38	# include <VBox/vmm/rem.h>
39	#endif
40	#include "HMInternal.h"
41	#include <VBox/vmm/vm.h>
42	#include <VBox/vmm/hmvmxinline.h>
43	#include "HMVMXR0.h"
44	#include "dtrace/VBoxVMM.h"
45
46	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
47	#ifdef DEBUG_ramshankar
48	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
49	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
50	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
51	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
52	# define HMVMX_ALWAYS_TRAP_PF
53	# define HMVMX_ALWAYS_FLUSH_TLB
54	# define HMVMX_ALWAYS_SWAP_EFER
55	#endif
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	/** Use the function table. */
62	#define HMVMX_USE_FUNCTION_TABLE
63
64	/** Determine which tagged-TLB flush handler to use. */
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70	/** @name HMVMX_READ_XXX
71	* Flags to skip redundant reads of some common VMCS fields that are not part of
72	* the guest-CPU or VCPU state but are needed while handling VM-exits.
73	*/
74	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
75	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
76	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
77	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
78	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
79	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
80	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
81	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
82	/** @} */
83
84	/**
85	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
86	* guest using hardware-assisted VMX.
87	*
88	* This excludes state like GPRs (other than RSP) which are always are
89	* swapped and restored across the world-switch and also registers like EFER,
90	* MSR which cannot be modified by the guest without causing a VM-exit.
91	*/
92	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
93	\| CPUMCTX_EXTRN_RFLAGS \
94	\| CPUMCTX_EXTRN_RSP \
95	\| CPUMCTX_EXTRN_SREG_MASK \
96	\| CPUMCTX_EXTRN_TABLE_MASK \
97	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
98	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
99	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
100	\| CPUMCTX_EXTRN_TSC_AUX \
101	\| CPUMCTX_EXTRN_OTHER_MSRS \
102	\| CPUMCTX_EXTRN_CR0 \
103	\| CPUMCTX_EXTRN_CR3 \
104	\| CPUMCTX_EXTRN_CR4 \
105	\| CPUMCTX_EXTRN_DR7 \
106	\| CPUMCTX_EXTRN_HM_VMX_MASK)
107
108	/**
109	* Exception bitmap mask for real-mode guests (real-on-v86).
110	*
111	* We need to intercept all exceptions manually except:
112	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
113	* due to bugs in Intel CPUs.
114	* - \#PF need not be intercepted even in real-mode if we have nested paging
115	* support.
116	*/
117	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
118	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
119	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
120	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
121	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
122	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
123	\| RT_BIT(X86_XCPT_XF))
124
125	/** Maximum VM-instruction error number. */
126	#define HMVMX_INSTR_ERROR_MAX 28
127
128	/** Profiling macro. */
129	#ifdef HM_PROFILE_EXIT_DISPATCH
130	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
131	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
132	#else
133	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
134	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
135	#endif
136
137	/** Assert that preemption is disabled or covered by thread-context hooks. */
138	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
139	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
140
141	/** Assert that we haven't migrated CPUs when thread-context hooks are not
142	* used. */
143	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
144	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
145	("Illegal migration! Entered on CPU %u Current %u\n", \
146	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
147
148	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
149	* context. */
150	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
151	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
152	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
153
154	/** Helper macro for VM-exit handlers called unexpectedly. */
155	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
156	do { \
157	(a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
158	return VERR_VMX_UNEXPECTED_EXIT; \
159	} while (0)
160
161	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
162	/** Macro that does the necessary privilege checks and intercepted VM-exits for
163	* guests that attempted to execute a VMX instruction. */
164	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
165	do \
166	{ \
167	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
168	if (rcStrictTmp == VINF_SUCCESS) \
169	{ /* likely */ } \
170	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
171	{ \
172	Assert((a_pVCpu)->hm.s.Event.fPending); \
173	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
174	return VINF_SUCCESS; \
175	} \
176	else \
177	{ \
178	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
179	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
180	} \
181	} while (0)
182
183	/** Macro that decodes a memory operand for an instruction VM-exit. */
184	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
185	do \
186	{ \
187	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
188	(a_pGCPtrEffAddr)); \
189	if (rcStrictTmp == VINF_SUCCESS) \
190	{ /* likely */ } \
191	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
192	{ \
193	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
194	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
195	NOREF(uXcptTmp); \
196	return VINF_SUCCESS; \
197	} \
198	else \
199	{ \
200	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
201	return rcStrictTmp; \
202	} \
203	} while (0)
204
205	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
206
207
208	/*********************************************************************************************************************************
209	* Structures and Typedefs *
210	*********************************************************************************************************************************/
211	/**
212	* VMX transient state.
213	*
214	* A state structure for holding miscellaneous information across
215	* VMX non-root operation and restored after the transition.
216	*/
217	typedef struct VMXTRANSIENT
218	{
219	/** The host's rflags/eflags. */
220	RTCCUINTREG fEFlags;
221	#if HC_ARCH_BITS == 32
222	uint32_t u32Alignment0;
223	#endif
224	/** The guest's TPR value used for TPR shadowing. */
225	uint8_t u8GuestTpr;
226	/** Alignment. */
227	uint8_t abAlignment0[7];
228
229	/** The basic VM-exit reason. */
230	uint16_t uExitReason;
231	/** Alignment. */
232	uint16_t u16Alignment0;
233	/** The VM-exit interruption error code. */
234	uint32_t uExitIntErrorCode;
235	/** The VM-exit exit code qualification. */
236	uint64_t uExitQual;
237	/** The Guest-linear address. */
238	uint64_t uGuestLinearAddr;
239
240	/** The VM-exit interruption-information field. */
241	uint32_t uExitIntInfo;
242	/** The VM-exit instruction-length field. */
243	uint32_t cbInstr;
244	/** The VM-exit instruction-information field. */
245	VMXEXITINSTRINFO ExitInstrInfo;
246	/** Whether the VM-entry failed or not. */
247	bool fVMEntryFailed;
248	/** Whether we are currently executing a nested-guest. */
249	bool fIsNestedGuest;
250	/** Alignment. */
251	uint8_t abAlignment1[2];
252
253	/** The VM-entry interruption-information field. */
254	uint32_t uEntryIntInfo;
255	/** The VM-entry exception error code field. */
256	uint32_t uEntryXcptErrorCode;
257	/** The VM-entry instruction length field. */
258	uint32_t cbEntryInstr;
259
260	/** IDT-vectoring information field. */
261	uint32_t uIdtVectoringInfo;
262	/** IDT-vectoring error code. */
263	uint32_t uIdtVectoringErrorCode;
264
265	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
266	uint32_t fVmcsFieldsRead;
267
268	/** Whether the guest debug state was active at the time of VM-exit. */
269	bool fWasGuestDebugStateActive;
270	/** Whether the hyper debug state was active at the time of VM-exit. */
271	bool fWasHyperDebugStateActive;
272	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
273	bool fUpdatedTscOffsettingAndPreemptTimer;
274	/** Whether the VM-exit was caused by a page-fault during delivery of a
275	* contributory exception or a page-fault. */
276	bool fVectoringDoublePF;
277	/** Whether the VM-exit was caused by a page-fault during delivery of an
278	* external interrupt or NMI. */
279	bool fVectoringPF;
280	bool afAlignment0[3];
281
282	/** The VMCS info. object. */
283	PVMXVMCSINFO pVmcsInfo;
284	} VMXTRANSIENT;
285	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
286	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
287	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
288	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
289	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
290	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
291	/** Pointer to VMX transient state. */
292	typedef VMXTRANSIENT *PVMXTRANSIENT;
293
294	/**
295	* Memory operand read or write access.
296	*/
297	typedef enum VMXMEMACCESS
298	{
299	VMXMEMACCESS_READ = 0,
300	VMXMEMACCESS_WRITE = 1
301	} VMXMEMACCESS;
302
303	/**
304	* VMX VM-exit handler.
305	*
306	* @returns Strict VBox status code (i.e. informational status codes too).
307	* @param pVCpu The cross context virtual CPU structure.
308	* @param pVmxTransient The VMX-transient structure.
309	*/
310	#ifndef HMVMX_USE_FUNCTION_TABLE
311	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
312	#else
313	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
314	/** Pointer to VM-exit handler. */
315	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
316	#endif
317
318	/**
319	* VMX VM-exit handler, non-strict status code.
320	*
321	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
322	*
323	* @returns VBox status code, no informational status code returned.
324	* @param pVCpu The cross context virtual CPU structure.
325	* @param pVmxTransient The VMX-transient structure.
326	*
327	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
328	* use of that status code will be replaced with VINF_EM_SOMETHING
329	* later when switching over to IEM.
330	*/
331	#ifndef HMVMX_USE_FUNCTION_TABLE
332	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
333	#else
334	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
335	#endif
336
337
338	/*********************************************************************************************************************************
339	* Internal Functions *
340	*********************************************************************************************************************************/
341	#ifndef HMVMX_USE_FUNCTION_TABLE
342	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
343	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
344	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
345	#else
346	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
347	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
348	#endif
349	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
350	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
351	#endif
352
353	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
354	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
355	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
356	#endif
357
358	/** @name VM-exit handlers.
359	* @{
360	*/
361	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
362	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
363	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
364	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
365	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
366	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
367	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
368	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
369	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
370	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
371	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
372	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
373	static FNVMXEXITHANDLER hmR0VmxExitHlt;
374	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
375	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
376	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
377	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
378	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
379	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
380	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
381	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
382	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
383	static FNVMXEXITHANDLER hmR0VmxExitVmread;
384	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
385	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
386	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
387	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
388	#endif
389	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
390	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
391	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
392	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
393	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
394	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
395	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
396	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
397	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
398	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
399	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
400	static FNVMXEXITHANDLER hmR0VmxExitMwait;
401	static FNVMXEXITHANDLER hmR0VmxExitMtf;
402	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
403	static FNVMXEXITHANDLER hmR0VmxExitPause;
404	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
405	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
406	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
407	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
408	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
409	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
410	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
411	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
412	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
413	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
414	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
415	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
416	/** @} */
417
418	/** @name Helpers for hardware exceptions VM-exit handlers.
419	* @{
420	*/
421	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
422	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
423	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
424	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
425	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
426	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
427	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
428	/** @} */
429
430
431	/*********************************************************************************************************************************
432	* Global Variables *
433	*********************************************************************************************************************************/
434	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
435	static const uint32_t g_aVmcsCacheSegBase[] =
436	{
437	VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
438	VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
439	VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
440	VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
441	VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
442	VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
443	};
444	AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
445	#endif
446	static const uint32_t g_aVmcsSegBase[] =
447	{
448	VMX_VMCS_GUEST_ES_BASE,
449	VMX_VMCS_GUEST_CS_BASE,
450	VMX_VMCS_GUEST_SS_BASE,
451	VMX_VMCS_GUEST_DS_BASE,
452	VMX_VMCS_GUEST_FS_BASE,
453	VMX_VMCS_GUEST_GS_BASE
454	};
455	static const uint32_t g_aVmcsSegSel[] =
456	{
457	VMX_VMCS16_GUEST_ES_SEL,
458	VMX_VMCS16_GUEST_CS_SEL,
459	VMX_VMCS16_GUEST_SS_SEL,
460	VMX_VMCS16_GUEST_DS_SEL,
461	VMX_VMCS16_GUEST_FS_SEL,
462	VMX_VMCS16_GUEST_GS_SEL
463	};
464	static const uint32_t g_aVmcsSegLimit[] =
465	{
466	VMX_VMCS32_GUEST_ES_LIMIT,
467	VMX_VMCS32_GUEST_CS_LIMIT,
468	VMX_VMCS32_GUEST_SS_LIMIT,
469	VMX_VMCS32_GUEST_DS_LIMIT,
470	VMX_VMCS32_GUEST_FS_LIMIT,
471	VMX_VMCS32_GUEST_GS_LIMIT
472	};
473	static const uint32_t g_aVmcsSegAttr[] =
474	{
475	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
476	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
477	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
478	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
479	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
480	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
481	};
482	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
483	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
484	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
485	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
486
487	#ifdef HMVMX_USE_FUNCTION_TABLE
488	/**
489	* VMX_EXIT dispatch table.
490	*/
491	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
492	{
493	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
494	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
495	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
496	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
497	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
498	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
499	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
500	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
501	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
502	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
503	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
504	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
505	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
506	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
507	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
508	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
509	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
510	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
511	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
512	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
513	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
514	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
515	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
516	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
517	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
518	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
519	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
520	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
521	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
522	#else
523	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
524	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
525	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
526	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
527	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
528	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
529	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
530	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
531	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
532	#endif
533	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
534	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
535	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
536	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
537	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
538	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
539	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
540	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
541	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
542	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
543	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
544	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
545	/* 40 UNDEFINED */ hmR0VmxExitPause,
546	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
547	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
548	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
549	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
550	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
551	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitXdtrAccess,
552	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitXdtrAccess,
553	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
554	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
555	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
556	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
557	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
558	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
559	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
560	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
561	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
562	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
563	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
564	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
565	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
566	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
567	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
568	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
569	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
570	};
571	#endif /* HMVMX_USE_FUNCTION_TABLE */
572
573	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
574	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
575	{
576	/* 0 */ "(Not Used)",
577	/* 1 */ "VMCALL executed in VMX root operation.",
578	/* 2 */ "VMCLEAR with invalid physical address.",
579	/* 3 */ "VMCLEAR with VMXON pointer.",
580	/* 4 */ "VMLAUNCH with non-clear VMCS.",
581	/* 5 */ "VMRESUME with non-launched VMCS.",
582	/* 6 */ "VMRESUME after VMXOFF",
583	/* 7 */ "VM-entry with invalid control fields.",
584	/* 8 */ "VM-entry with invalid host state fields.",
585	/* 9 */ "VMPTRLD with invalid physical address.",
586	/* 10 */ "VMPTRLD with VMXON pointer.",
587	/* 11 */ "VMPTRLD with incorrect revision identifier.",
588	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
589	/* 13 */ "VMWRITE to read-only VMCS component.",
590	/* 14 */ "(Not Used)",
591	/* 15 */ "VMXON executed in VMX root operation.",
592	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
593	/* 17 */ "VM-entry with non-launched executing VMCS.",
594	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
595	/* 19 */ "VMCALL with non-clear VMCS.",
596	/* 20 */ "VMCALL with invalid VM-exit control fields.",
597	/* 21 */ "(Not Used)",
598	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
599	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
600	/* 24 */ "VMCALL with invalid SMM-monitor features.",
601	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
602	/* 26 */ "VM-entry with events blocked by MOV SS.",
603	/* 27 */ "(Not Used)",
604	/* 28 */ "Invalid operand to INVEPT/INVVPID."
605	};
606	#endif /* VBOX_STRICT */
607
608
609	/**
610	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
611	*
612	* Any bit set in this mask is owned by the host/hypervisor and would cause a
613	* VM-exit when modified by the guest.
614	*
615	* @returns The static CR0 guest/host mask.
616	* @param pVCpu The cross context virtual CPU structure.
617	*/
618	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PCVMCPU pVCpu)
619	{
620	/*
621	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
622	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
623	*/
624	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
625	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
626	* and @bugref{6944}. */
627	PVM pVM = pVCpu->CTX_SUFF(pVM);
628	return ( X86_CR0_PE
629	\| X86_CR0_NE
630	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
631	\| X86_CR0_PG
632	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
633	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
634	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
635	}
636
637
638	/**
639	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
640	*
641	* Any bit set in this mask is owned by the host/hypervisor and would cause a
642	* VM-exit when modified by the guest.
643	*
644	* @returns The static CR4 guest/host mask.
645	* @param pVCpu The cross context virtual CPU structure.
646	*/
647	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PCVMCPU pVCpu)
648	{
649	/*
650	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
651	* these bits are reserved on hardware that does not support them. Since the
652	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
653	* these bits and handle it depending on whether we expose them to the guest.
654	*/
655	PVM pVM = pVCpu->CTX_SUFF(pVM);
656	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
657	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
658	return ( X86_CR4_VMXE
659	\| X86_CR4_VME
660	\| X86_CR4_PAE
661	\| X86_CR4_PGE
662	\| X86_CR4_PSE
663	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
664	\| (fPcid ? X86_CR4_PCIDE : 0));
665	}
666
667
668	/**
669	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
670	* area.
671	*
672	* @returns @c true if it's different, @c false otherwise.
673	* @param pVmcsInfo The VMCS info. object.
674	*/
675	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
676	{
677	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
678	&& pVmcsInfo->pvGuestMsrStore);
679	}
680
681
682	/**
683	* Adds one or more exceptions to the exception bitmap and commits it to the current
684	* VMCS.
685	*
686	* @returns VBox status code.
687	* @param pVmxTransient The VMX-transient structure.
688	* @param uXcptMask The exception(s) to add.
689	*/
690	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
691	{
692	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
693	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
694	if ((uXcptBitmap & uXcptMask) != uXcptMask)
695	{
696	uXcptBitmap \|= uXcptMask;
697	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
698	AssertRCReturn(rc, rc);
699	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
700	}
701	return VINF_SUCCESS;
702	}
703
704
705	/**
706	* Adds an exception to the exception bitmap and commits it to the current VMCS.
707	*
708	* @returns VBox status code.
709	* @param pVmxTransient The VMX-transient structure.
710	* @param uXcpt The exception to add.
711	*/
712	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
713	{
714	Assert(uXcpt <= X86_XCPT_LAST);
715	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
716	}
717
718
719	/**
720	* Remove one or more exceptions from the exception bitmap and commits it to the
721	* current VMCS.
722	*
723	* This takes care of not removing the exception intercept if a nested-guest
724	* requires the exception to be intercepted.
725	*
726	* @returns VBox status code.
727	* @param pVCpu The cross context virtual CPU structure.
728	* @param pVmxTransient The VMX-transient structure.
729	* @param uXcptMask The exception(s) to remove.
730	*/
731	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
732	{
733	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
734	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
735	if (u32XcptBitmap & uXcptMask)
736	{
737	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
738	if (!pVmxTransient->fIsNestedGuest)
739	{ /* likely */ }
740	else
741	{
742	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
743	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
744	}
745	#endif
746	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
747	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
748	\| RT_BIT(X86_XCPT_DE)
749	\| RT_BIT(X86_XCPT_NM)
750	\| RT_BIT(X86_XCPT_TS)
751	\| RT_BIT(X86_XCPT_UD)
752	\| RT_BIT(X86_XCPT_NP)
753	\| RT_BIT(X86_XCPT_SS)
754	\| RT_BIT(X86_XCPT_GP)
755	\| RT_BIT(X86_XCPT_PF)
756	\| RT_BIT(X86_XCPT_MF));
757	#elif defined(HMVMX_ALWAYS_TRAP_PF)
758	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
759	#endif
760	if (uXcptMask)
761	{
762	/* Validate we are not removing any essential exception intercepts. */
763	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
764	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
765	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
766
767	/* Remove it from the exception bitmap. */
768	u32XcptBitmap &= ~uXcptMask;
769
770	/* Commit and update the cache if necessary. */
771	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
772	{
773	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
774	AssertRCReturn(rc, rc);
775	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
776	}
777	}
778	}
779	return VINF_SUCCESS;
780	}
781
782
783	/**
784	* Remove an exceptions from the exception bitmap and commits it to the current
785	* VMCS.
786	*
787	* @returns VBox status code.
788	* @param pVCpu The cross context virtual CPU structure.
789	* @param pVmxTransient The VMX-transient structure.
790	* @param uXcpt The exception to remove.
791	*/
792	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
793	{
794	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
795	}
796
797
798	/**
799	* Loads the VMCS specified by the VMCS info. object.
800	*
801	* @returns VBox status code.
802	* @param pVmcsInfo The VMCS info. object.
803	*/
804	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
805	{
806	Assert(pVmcsInfo);
807	Assert(pVmcsInfo->HCPhysVmcs);
808	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
809
810	if (pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_CLEAR)
811	{
812	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
813	if (RT_SUCCESS(rc))
814	{
815	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
816	return VINF_SUCCESS;
817	}
818	return rc;
819	}
820	return VERR_VMX_INVALID_VMCS_LAUNCH_STATE;
821	}
822
823
824	/**
825	* Clears the VMCS specified by the VMCS info. object.
826	*
827	* @returns VBox status code.
828	* @param pVmcsInfo The VMCS info. object.
829	*/
830	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
831	{
832	Assert(pVmcsInfo);
833	Assert(pVmcsInfo->HCPhysVmcs);
834	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
835
836	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
837	if (RT_SUCCESS(rc))
838	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
839	return rc;
840	}
841
842
843	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
844	/**
845	* Switches the current VMCS to the one specified.
846	*
847	* @returns VBox status code.
848	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
849	* @param pVmcsInfoTo The VMCS info. object we are switching to.
850	*
851	* @remarks Called with interrupts disabled.
852	*/
853	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
854	{
855	Assert(pVmcsInfoFrom);
856	Assert(pVmcsInfoTo);
857
858	/*
859	* Clear the VMCS we are switching out if it has not already been cleared.
860	* This will sync any CPU internal data back to the VMCS.
861	*/
862	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
863	{
864	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
865	if (RT_SUCCESS(rc))
866	{ /* likely */ }
867	else
868	return rc;
869	}
870
871	/*
872	* Clear the VMCS we are switching to if it has not already been cleared.
873	* This will initialize the VMCS launch state to "clear" required for loading it.
874	*
875	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
876	*/
877	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
878	{
879	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
880	if (RT_SUCCESS(rc))
881	{ /* likely */ }
882	else
883	return rc;
884	}
885
886	/*
887	* Finally, load the VMCS we are switching to.
888	*/
889	return hmR0VmxLoadVmcs(pVmcsInfoTo);
890	}
891	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
892
893
894	/**
895	* Updates the VM's last error record.
896	*
897	* If there was a VMX instruction error, reads the error data from the VMCS and
898	* updates VCPU's last error record as well.
899	*
900	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
901	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
902	* VERR_VMX_INVALID_VMCS_FIELD.
903	* @param rc The error code.
904	*/
905	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
906	{
907	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
908	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
909	{
910	AssertPtrReturnVoid(pVCpu);
911	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
912	}
913	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
914	}
915
916
917	#ifdef VBOX_STRICT
918	/**
919	* Reads the VM-entry interruption-information field from the VMCS into the VMX
920	* transient structure.
921	*
922	* @returns VBox status code.
923	* @param pVmxTransient The VMX-transient structure.
924	*
925	* @remarks No-long-jump zone!!!
926	*/
927	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
928	{
929	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
930	AssertRCReturn(rc, rc);
931	return VINF_SUCCESS;
932	}
933
934
935	/**
936	* Reads the VM-entry exception error code field from the VMCS into
937	* the VMX transient structure.
938	*
939	* @returns VBox status code.
940	* @param pVmxTransient The VMX-transient structure.
941	*
942	* @remarks No-long-jump zone!!!
943	*/
944	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
945	{
946	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
947	AssertRCReturn(rc, rc);
948	return VINF_SUCCESS;
949	}
950
951
952	/**
953	* Reads the VM-entry exception error code field from the VMCS into
954	* the VMX transient structure.
955	*
956	* @returns VBox status code.
957	* @param pVmxTransient The VMX-transient structure.
958	*
959	* @remarks No-long-jump zone!!!
960	*/
961	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
962	{
963	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
964	AssertRCReturn(rc, rc);
965	return VINF_SUCCESS;
966	}
967	#endif /* VBOX_STRICT */
968
969
970	/**
971	* Reads the VM-exit interruption-information field from the VMCS into the VMX
972	* transient structure.
973	*
974	* @returns VBox status code.
975	* @param pVmxTransient The VMX-transient structure.
976	*/
977	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
978	{
979	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
980	{
981	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
982	AssertRCReturn(rc,rc);
983	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
984	}
985	return VINF_SUCCESS;
986	}
987
988
989	/**
990	* Reads the VM-exit interruption error code from the VMCS into the VMX
991	* transient structure.
992	*
993	* @returns VBox status code.
994	* @param pVmxTransient The VMX-transient structure.
995	*/
996	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
997	{
998	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
999	{
1000	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1001	AssertRCReturn(rc, rc);
1002	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1003	}
1004	return VINF_SUCCESS;
1005	}
1006
1007
1008	/**
1009	* Reads the VM-exit instruction length field from the VMCS into the VMX
1010	* transient structure.
1011	*
1012	* @returns VBox status code.
1013	* @param pVmxTransient The VMX-transient structure.
1014	*/
1015	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1016	{
1017	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1018	{
1019	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1020	AssertRCReturn(rc, rc);
1021	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1022	}
1023	return VINF_SUCCESS;
1024	}
1025
1026
1027	/**
1028	* Reads the VM-exit instruction-information field from the VMCS into
1029	* the VMX transient structure.
1030	*
1031	* @returns VBox status code.
1032	* @param pVmxTransient The VMX-transient structure.
1033	*/
1034	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1035	{
1036	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1037	{
1038	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1039	AssertRCReturn(rc, rc);
1040	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1041	}
1042	return VINF_SUCCESS;
1043	}
1044
1045
1046	/**
1047	* Reads the VM-exit Qualification from the VMCS into the VMX transient structure.
1048	*
1049	* @returns VBox status code.
1050	* @param pVCpu The cross context virtual CPU structure of the
1051	* calling EMT. (Required for the VMCS cache case.)
1052	* @param pVmxTransient The VMX-transient structure.
1053	*/
1054	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1055	{
1056	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1057	{
1058	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1059	AssertRCReturn(rc, rc);
1060	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1061	}
1062	return VINF_SUCCESS;
1063	}
1064
1065
1066	/**
1067	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1068	*
1069	* @returns VBox status code.
1070	* @param pVCpu The cross context virtual CPU structure of the
1071	* calling EMT. (Required for the VMCS cache case.)
1072	* @param pVmxTransient The VMX-transient structure.
1073	*/
1074	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1075	{
1076	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1077	{
1078	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1079	AssertRCReturn(rc, rc);
1080	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1081	}
1082	return VINF_SUCCESS;
1083	}
1084
1085
1086	/**
1087	* Reads the IDT-vectoring information field from the VMCS into the VMX
1088	* transient structure.
1089	*
1090	* @returns VBox status code.
1091	* @param pVmxTransient The VMX-transient structure.
1092	*
1093	* @remarks No-long-jump zone!!!
1094	*/
1095	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1096	{
1097	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1098	{
1099	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1100	AssertRCReturn(rc, rc);
1101	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1102	}
1103	return VINF_SUCCESS;
1104	}
1105
1106
1107	/**
1108	* Reads the IDT-vectoring error code from the VMCS into the VMX
1109	* transient structure.
1110	*
1111	* @returns VBox status code.
1112	* @param pVmxTransient The VMX-transient structure.
1113	*/
1114	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1115	{
1116	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1117	{
1118	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1119	AssertRCReturn(rc, rc);
1120	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1121	}
1122	return VINF_SUCCESS;
1123	}
1124
1125
1126	/**
1127	* Enters VMX root mode operation on the current CPU.
1128	*
1129	* @returns VBox status code.
1130	* @param pVM The cross context VM structure. Can be
1131	* NULL, after a resume.
1132	* @param HCPhysCpuPage Physical address of the VMXON region.
1133	* @param pvCpuPage Pointer to the VMXON region.
1134	*/
1135	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1136	{
1137	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1138	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1139	Assert(pvCpuPage);
1140	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1141
1142	if (pVM)
1143	{
1144	/* Write the VMCS revision identifier to the VMXON region. */
1145	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1146	}
1147
1148	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1149	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1150
1151	/* Enable the VMX bit in CR4 if necessary. */
1152	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1153
1154	/* Enter VMX root mode. */
1155	int rc = VMXEnable(HCPhysCpuPage);
1156	if (RT_FAILURE(rc))
1157	{
1158	if (!(uOldCr4 & X86_CR4_VMXE))
1159	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1160
1161	if (pVM)
1162	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1163	}
1164
1165	/* Restore interrupts. */
1166	ASMSetFlags(fEFlags);
1167	return rc;
1168	}
1169
1170
1171	/**
1172	* Exits VMX root mode operation on the current CPU.
1173	*
1174	* @returns VBox status code.
1175	*/
1176	static int hmR0VmxLeaveRootMode(void)
1177	{
1178	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1179
1180	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1181	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1182
1183	/* If we're for some reason not in VMX root mode, then don't leave it. */
1184	RTCCUINTREG const uHostCR4 = ASMGetCR4();
1185
1186	int rc;
1187	if (uHostCR4 & X86_CR4_VMXE)
1188	{
1189	/* Exit VMX root mode and clear the VMX bit in CR4. */
1190	VMXDisable();
1191	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1192	rc = VINF_SUCCESS;
1193	}
1194	else
1195	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1196
1197	/* Restore interrupts. */
1198	ASMSetFlags(fEFlags);
1199	return rc;
1200	}
1201
1202
1203	/**
1204	* Allocates and maps a physically contiguous page. The allocated page is
1205	* zero'd out (used by various VT-x structures).
1206	*
1207	* @returns IPRT status code.
1208	* @param pMemObj Pointer to the ring-0 memory object.
1209	* @param ppVirt Where to store the virtual address of the
1210	* allocation.
1211	* @param pHCPhys Where to store the physical address of the
1212	* allocation.
1213	*/
1214	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1215	{
1216	AssertPtr(pMemObj);
1217	AssertPtr(ppVirt);
1218	AssertPtr(pHCPhys);
1219	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1220	if (RT_FAILURE(rc))
1221	return rc;
1222	ppVirt = RTR0MemObjAddress(pMemObj);
1223	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1224	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1225	return VINF_SUCCESS;
1226	}
1227
1228
1229	/**
1230	* Frees and unmaps an allocated, physical page.
1231	*
1232	* @param pMemObj Pointer to the ring-0 memory object.
1233	* @param ppVirt Where to re-initialize the virtual address of
1234	* allocation as 0.
1235	* @param pHCPhys Where to re-initialize the physical address of the
1236	* allocation as 0.
1237	*/
1238	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1239	{
1240	AssertPtr(pMemObj);
1241	AssertPtr(ppVirt);
1242	AssertPtr(pHCPhys);
1243	/* NULL is valid, accepted and ignored by the free function below. */
1244	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1245	*pMemObj = NIL_RTR0MEMOBJ;
1246	*ppVirt = NULL;
1247	*pHCPhys = NIL_RTHCPHYS;
1248	}
1249
1250
1251	/**
1252	* Initializes a VMCS info. object.
1253	*
1254	* @param pVmcsInfo The VMCS info. object.
1255	*/
1256	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1257	{
1258	RT_ZERO(*pVmcsInfo);
1259
1260	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1261	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1262	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1263	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1264	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1265	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1266	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1267	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1268	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1269	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1270	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1271	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1272	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1273	}
1274
1275
1276	/**
1277	* Frees the VT-x structures for a VMCS info. object.
1278	*
1279	* @param pVM The cross context VM structure.
1280	* @param pVmcsInfo The VMCS info. object.
1281	*/
1282	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1283	{
1284	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1285
1286	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1287	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1288
1289	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1290	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1291	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1292
1293	hmR0VmxInitVmcsInfo(pVmcsInfo);
1294	}
1295
1296
1297	/**
1298	* Allocates the VT-x structures for a VMCS info. object.
1299	*
1300	* @returns VBox status code.
1301	* @param pVCpu The cross context virtual CPU structure.
1302	* @param pVmcsInfo The VMCS info. object.
1303	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1304	*/
1305	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1306	{
1307	PVM pVM = pVCpu->CTX_SUFF(pVM);
1308
1309	/* Allocate the guest VM control structure (VMCS). */
1310	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1311	if (RT_SUCCESS(rc))
1312	{
1313	if (!fIsNstGstVmcs)
1314	{
1315	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1316	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1317	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1318	{
1319	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1320	NULL /* pR3Ptr /, NULL / pRCPtr */);
1321	}
1322	}
1323	else
1324	{
1325	Assert(pVmcsInfo->HCPhysVirtApic == NIL_RTHCPHYS);
1326	Assert(!pVmcsInfo->pbVirtApic);
1327	}
1328
1329	if (RT_SUCCESS(rc))
1330	{
1331	/*
1332	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1333	* transparent accesses of specific MSRs.
1334	*
1335	* If the condition for enabling MSR bitmaps changes here, don't forget to
1336	* update HMIsMsrBitmapActive().
1337	*
1338	* We don't share MSR bitmaps between the guest and nested-guest as we then
1339	* don't need to care about carefully restoring the guest MSR bitmap.
1340	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1341	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1342	*/
1343	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1344	{
1345	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1346	if (RT_SUCCESS(rc))
1347	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1348	}
1349
1350	if (RT_SUCCESS(rc))
1351	{
1352	/*
1353	* Allocate the VM-entry MSR-load area for the guest MSRs.
1354	*
1355	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1356	* the guest and nested-guest.
1357	*/
1358	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1359	&pVmcsInfo->HCPhysGuestMsrLoad);
1360	if (RT_SUCCESS(rc))
1361	{
1362	/*
1363	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1364	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1365	*/
1366	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1367	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1368	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1369
1370	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1371	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1372	&pVmcsInfo->HCPhysHostMsrLoad);
1373	}
1374	}
1375	}
1376	}
1377
1378	return rc;
1379	}
1380
1381
1382	/**
1383	* Free all VT-x structures for the VM.
1384	*
1385	* @returns IPRT status code.
1386	* @param pVM The cross context VM structure.
1387	*/
1388	static void hmR0VmxStructsFree(PVM pVM)
1389	{
1390	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1391	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1392	#endif
1393	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1394
1395	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1396	{
1397	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1398	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1399	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1400	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1401	if (pVM->cpum.ro.GuestFeatures.fVmx)
1402	{
1403	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1404	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1405	}
1406	#endif
1407	}
1408	}
1409
1410
1411	/**
1412	* Allocate all VT-x structures for the VM.
1413	*
1414	* @returns IPRT status code.
1415	* @param pVM The cross context VM structure.
1416	*/
1417	static int hmR0VmxStructsAlloc(PVM pVM)
1418	{
1419	/*
1420	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1421	* The VMCS size cannot be more than 4096 bytes.
1422	*
1423	* See Intel spec. Appendix A.1 "Basic VMX Information".
1424	*/
1425	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1426	if (cbVmcs <= X86_PAGE_4K_SIZE)
1427	{ /* likely */ }
1428	else
1429	{
1430	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1431	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1432	}
1433
1434	/*
1435	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1436	*/
1437	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1438	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1439	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1440	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1441	#endif
1442
1443	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1444	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1445	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1446
1447	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1448	{
1449	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1450	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1451	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1452	}
1453
1454	/*
1455	* Allocate per-VM VT-x structures.
1456	*/
1457	int rc = VINF_SUCCESS;
1458	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1459	/* Allocate crash-dump magic scratch page. */
1460	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1461	if (RT_FAILURE(rc))
1462	{
1463	hmR0VmxStructsFree(pVM);
1464	return rc;
1465	}
1466	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1467	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1468	#endif
1469
1470	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1471	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1472	{
1473	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1474	&pVM->hm.s.vmx.HCPhysApicAccess);
1475	if (RT_FAILURE(rc))
1476	{
1477	hmR0VmxStructsFree(pVM);
1478	return rc;
1479	}
1480	}
1481
1482	/*
1483	* Initialize per-VCPU VT-x structures.
1484	*/
1485	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1486	{
1487	/* Allocate the guest VMCS structures. */
1488	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1489	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
1490	if (RT_SUCCESS(rc))
1491	{
1492	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1493	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
1494	if (pVM->cpum.ro.GuestFeatures.fVmx)
1495	{
1496	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
1497	if (RT_SUCCESS(rc))
1498	{ /* likely */ }
1499	else
1500	break;
1501	}
1502	#endif
1503	}
1504	else
1505	break;
1506	}
1507
1508	if (RT_FAILURE(rc))
1509	{
1510	hmR0VmxStructsFree(pVM);
1511	return rc;
1512	}
1513
1514	return VINF_SUCCESS;
1515	}
1516
1517
1518	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1519	/**
1520	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
1521	*
1522	* @returns @c true if the MSR is intercepted, @c false otherwise.
1523	* @param pvMsrBitmap The MSR bitmap.
1524	* @param offMsr The MSR byte offset.
1525	* @param iBit The bit offset from the byte offset.
1526	*/
1527	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
1528	{
1529	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
1530	Assert(pbMsrBitmap);
1531	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
1532	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
1533	}
1534	#endif
1535
1536
1537	/**
1538	* Sets the permission bits for the specified MSR in the given MSR bitmap.
1539	*
1540	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
1541	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
1542	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
1543	* the read/write access of this MSR.
1544	*
1545	* @param pVCpu The cross context virtual CPU structure.
1546	* @param pVmcsInfo The VMCS info. object.
1547	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1548	* @param idMsr The MSR value.
1549	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
1550	* include both a read -and- a write permission!
1551	*
1552	* @sa HMGetVmxMsrPermission.
1553	*/
1554	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
1555	{
1556	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
1557	Assert(pbMsrBitmap);
1558	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
1559
1560	/*
1561	* MSR-bitmap Layout:
1562	* Byte index MSR range Interpreted as
1563	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1564	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1565	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1566	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1567	*
1568	* A bit corresponding to an MSR within the above range causes a VM-exit
1569	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
1570	* the MSR range, it always cause a VM-exit.
1571	*
1572	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
1573	*/
1574	uint16_t const offBitmapRead = 0;
1575	uint16_t const offBitmapWrite = 0x800;
1576	uint16_t offMsr;
1577	int32_t iBit;
1578	if (idMsr <= UINT32_C(0x00001fff))
1579	{
1580	offMsr = 0;
1581	iBit = idMsr;
1582	}
1583	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1584	{
1585	offMsr = 0x400;
1586	iBit = idMsr - UINT32_C(0xc0000000);
1587	}
1588	else
1589	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
1590
1591	/*
1592	* Set the MSR read permission.
1593	*/
1594	uint16_t const offMsrRead = offBitmapRead + offMsr;
1595	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
1596	if (fMsrpm & VMXMSRPM_ALLOW_RD)
1597	{
1598	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1599	bool const fClear = !fIsNstGstVmcs ? true
1600	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
1601	#else
1602	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1603	bool const fClear = true;
1604	#endif
1605	if (fClear)
1606	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
1607	}
1608	else
1609	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
1610
1611	/*
1612	* Set the MSR write permission.
1613	*/
1614	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
1615	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
1616	if (fMsrpm & VMXMSRPM_ALLOW_WR)
1617	{
1618	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1619	bool const fClear = !fIsNstGstVmcs ? true
1620	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
1621	#else
1622	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1623	bool const fClear = true;
1624	#endif
1625	if (fClear)
1626	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
1627	}
1628	else
1629	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
1630	}
1631
1632
1633	/**
1634	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1635	* area.
1636	*
1637	* @returns VBox status code.
1638	* @param pVCpu The cross context virtual CPU structure.
1639	* @param pVmcsInfo The VMCS info. object.
1640	* @param cMsrs The number of MSRs.
1641	*/
1642	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
1643	{
1644	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1645	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1646	if (RT_UNLIKELY(cMsrs >= cMaxSupportedMsrs))
1647	{
1648	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1649	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1650	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1651	}
1652
1653	/* Commit the MSR counts to the VMCS and update the cache. */
1654	int rc = VINF_SUCCESS;
1655	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
1656	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1657	if (pVmcsInfo->cExitMsrStore != cMsrs)
1658	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1659	if (pVmcsInfo->cExitMsrLoad != cMsrs)
1660	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1661	AssertRCReturn(rc, rc);
1662
1663	pVmcsInfo->cEntryMsrLoad = cMsrs;
1664	pVmcsInfo->cExitMsrStore = cMsrs;
1665	pVmcsInfo->cExitMsrLoad = cMsrs;
1666
1667	return VINF_SUCCESS;
1668	}
1669
1670
1671	/**
1672	* Adds a new (or updates the value of an existing) guest/host MSR
1673	* pair to be swapped during the world-switch as part of the
1674	* auto-load/store MSR area in the VMCS.
1675	*
1676	* @returns VBox status code.
1677	* @param pVCpu The cross context virtual CPU structure.
1678	* @param pVmxTransient The VMX-transient structure.
1679	* @param idMsr The MSR.
1680	* @param uGuestMsrValue Value of the guest MSR.
1681	* @param fSetReadWrite Whether to set the guest read/write access of this
1682	* MSR (thus not causing a VM-exit).
1683	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1684	* necessary.
1685	*/
1686	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
1687	bool fSetReadWrite, bool fUpdateHostMsr)
1688	{
1689	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1690	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1691	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1692	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1693	uint32_t i;
1694
1695	/* Paranoia. */
1696	Assert(pGuestMsrLoad);
1697
1698	/* Check if the MSR already exists in the VM-entry MSR-load area. */
1699	for (i = 0; i < cMsrs; i++)
1700	{
1701	if (pGuestMsrLoad->u32Msr == idMsr)
1702	break;
1703	pGuestMsrLoad++;
1704	}
1705
1706	bool fAdded = false;
1707	if (i == cMsrs)
1708	{
1709	/* The MSR does not exist, bump the MSR coun to make room for the new MSR. */
1710	++cMsrs;
1711	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1712	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
1713
1714	/* Set the guest to read/write this MSR without causing VM-exits. */
1715	if ( fSetReadWrite
1716	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
1717	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
1718
1719	fAdded = true;
1720	}
1721
1722	/* Update the MSR value for the newly added or already existing MSR. */
1723	pGuestMsrLoad->u32Msr = idMsr;
1724	pGuestMsrLoad->u64Value = uGuestMsrValue;
1725
1726	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
1727	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1728	{
1729	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1730	pGuestMsrStore += i;
1731	pGuestMsrStore->u32Msr = idMsr;
1732	pGuestMsrStore->u64Value = 0;
1733	}
1734
1735	/* Update the corresponding slot in the host MSR area. */
1736	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1737	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad && pHostMsr != pVmcsInfo->pvGuestMsrStore);
1738	pHostMsr += i;
1739	pHostMsr->u32Msr = idMsr;
1740
1741	/*
1742	* Only if the caller requests to update the host MSR value AND we've newly added the
1743	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
1744	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
1745	*
1746	* We do this for performance reasons since reading MSRs may be quite expensive.
1747	*/
1748	if ( fAdded
1749	&& fUpdateHostMsr)
1750	{
1751	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1752	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1753	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1754	}
1755	return VINF_SUCCESS;
1756	}
1757
1758
1759	/**
1760	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1761	* auto-load/store MSR area in the VMCS.
1762	*
1763	* @returns VBox status code.
1764	* @param pVCpu The cross context virtual CPU structure.
1765	* @param pVmxTransient The VMX-transient structure.
1766	* @param idMsr The MSR.
1767	*/
1768	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
1769	{
1770	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1771	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1772	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1773	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1774
1775	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
1776	for (uint32_t i = 0; i < cMsrs; i++)
1777	{
1778	/* Find the MSR. */
1779	if (pGuestMsrLoad->u32Msr == idMsr)
1780	{
1781	/* If it's the last MSR, simply reduce the count. */
1782	if (i == cMsrs - 1)
1783	{
1784	--cMsrs;
1785	break;
1786	}
1787
1788	/* Remove it by copying the last MSR in place of it, and reducing the count. */
1789	PVMXAUTOMSR pLastGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1790	pLastGuestMsrLoad += cMsrs - 1;
1791	pGuestMsrLoad->u32Msr = pLastGuestMsrLoad->u32Msr;
1792	pGuestMsrLoad->u64Value = pLastGuestMsrLoad->u64Value;
1793
1794	/* Remove it from the VM-exit MSR-store area if we are using a different page. */
1795	if (fSeparateExitMsrStorePage)
1796	{
1797	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1798	PVMXAUTOMSR pLastGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1799	pGuestMsrStore += i;
1800	pLastGuestMsrStore += cMsrs - 1;
1801	Assert(pGuestMsrStore->u32Msr == idMsr);
1802	pGuestMsrStore->u32Msr = pLastGuestMsrStore->u32Msr;
1803	pGuestMsrStore->u64Value = pLastGuestMsrStore->u64Value;
1804	}
1805
1806	/* Remove it from the VM-exit MSR-load area. */
1807	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1808	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1809	pHostMsr += i;
1810	pLastHostMsr += cMsrs - 1;
1811	Assert(pHostMsr->u32Msr == idMsr);
1812	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1813	pHostMsr->u64Value = pLastHostMsr->u64Value;
1814	--cMsrs;
1815	break;
1816	}
1817	pGuestMsrLoad++;
1818	}
1819
1820	/* Update the VMCS if the count changed (meaning the MSR was found). */
1821	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
1822	{
1823	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1824	AssertRCReturn(rc, rc);
1825
1826	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1827	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1828	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
1829
1830	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
1831	return VINF_SUCCESS;
1832	}
1833
1834	return VERR_NOT_FOUND;
1835	}
1836
1837
1838	/**
1839	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
1840	*
1841	* @returns @c true if found, @c false otherwise.
1842	* @param pVmcsInfo The VMCS info. object.
1843	* @param idMsr The MSR to find.
1844	*/
1845	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
1846	{
1847	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1848	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
1849	for (uint32_t i = 0; i < cMsrs; i++)
1850	{
1851	if (pGuestMsrLoad->u32Msr == idMsr)
1852	return true;
1853	pGuestMsrLoad++;
1854	}
1855	return false;
1856	}
1857
1858
1859	/**
1860	* Updates the value of all host MSRs in the VM-exit MSR-load area.
1861	*
1862	* @param pVCpu The cross context virtual CPU structure.
1863	* @param pVmcsInfo The VMCS info. object.
1864	*
1865	* @remarks No-long-jump zone!!!
1866	*/
1867	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
1868	{
1869	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1870	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
1871
1872	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1873	Assert(pHostMsrLoad);
1874
1875	for (uint32_t i = 0; i < cMsrs; i++, pHostMsrLoad++)
1876	{
1877	/*
1878	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1879	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1880	*/
1881	if (pHostMsrLoad->u32Msr == MSR_K6_EFER)
1882	pHostMsrLoad->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
1883	else
1884	pHostMsrLoad->u64Value = ASMRdMsr(pHostMsrLoad->u32Msr);
1885	}
1886	}
1887
1888
1889	/**
1890	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1891	* perform lazy restoration of the host MSRs while leaving VT-x.
1892	*
1893	* @param pVCpu The cross context virtual CPU structure.
1894	*
1895	* @remarks No-long-jump zone!!!
1896	*/
1897	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1898	{
1899	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1900
1901	/*
1902	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
1903	*/
1904	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1905	{
1906	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1907	#if HC_ARCH_BITS == 64
1908	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1909	{
1910	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
1911	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
1912	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
1913	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1914	}
1915	#endif
1916	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1917	}
1918	}
1919
1920
1921	/**
1922	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1923	* lazily while leaving VT-x.
1924	*
1925	* @returns true if it does, false otherwise.
1926	* @param pVCpu The cross context virtual CPU structure.
1927	* @param idMsr The MSR to check.
1928	*/
1929	static bool hmR0VmxIsLazyGuestMsr(PCVMCPU pVCpu, uint32_t idMsr)
1930	{
1931	NOREF(pVCpu);
1932	#if HC_ARCH_BITS == 64
1933	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1934	{
1935	switch (idMsr)
1936	{
1937	case MSR_K8_LSTAR:
1938	case MSR_K6_STAR:
1939	case MSR_K8_SF_MASK:
1940	case MSR_K8_KERNEL_GS_BASE:
1941	return true;
1942	}
1943	}
1944	#else
1945	RT_NOREF(pVCpu, idMsr);
1946	#endif
1947	return false;
1948	}
1949
1950
1951	/**
1952	* Loads a set of guests MSRs to allow read/passthru to the guest.
1953	*
1954	* The name of this function is slightly confusing. This function does NOT
1955	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1956	* common prefix for functions dealing with "lazy restoration" of the shared
1957	* MSRs.
1958	*
1959	* @param pVCpu The cross context virtual CPU structure.
1960	*
1961	* @remarks No-long-jump zone!!!
1962	*/
1963	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1964	{
1965	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1966	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1967
1968	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1969	#if HC_ARCH_BITS == 64
1970	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1971	{
1972	/*
1973	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1974	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1975	* we can skip a few MSR writes.
1976	*
1977	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1978	* guest MSR values in the guest-CPU context might be different to what's currently
1979	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1980	* CPU, see @bugref{8728}.
1981	*/
1982	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1983	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1984	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
1985	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
1986	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
1987	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
1988	{
1989	#ifdef VBOX_STRICT
1990	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1991	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1992	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1993	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1994	#endif
1995	}
1996	else
1997	{
1998	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1999	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2000	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2001	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2002	}
2003	}
2004	#endif
2005	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2006	}
2007
2008
2009	/**
2010	* Performs lazy restoration of the set of host MSRs if they were previously
2011	* loaded with guest MSR values.
2012	*
2013	* @param pVCpu The cross context virtual CPU structure.
2014	*
2015	* @remarks No-long-jump zone!!!
2016	* @remarks The guest MSRs should have been saved back into the guest-CPU
2017	* context by hmR0VmxImportGuestState()!!!
2018	*/
2019	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2020	{
2021	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2022	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2023
2024	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2025	{
2026	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2027	#if HC_ARCH_BITS == 64
2028	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2029	{
2030	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2031	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2032	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2033	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2034	}
2035	#endif
2036	}
2037	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2038	}
2039
2040
2041	/**
2042	* Verifies that our cached values of the VMCS fields are all consistent with
2043	* what's actually present in the VMCS.
2044	*
2045	* @returns VBox status code.
2046	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2047	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2048	* VMCS content. HMCPU error-field is
2049	* updated, see VMX_VCI_XXX.
2050	* @param pVCpu The cross context virtual CPU structure.
2051	* @param pVmcsInfo The VMCS info. object.
2052	*/
2053	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2054	{
2055	uint32_t u32Val;
2056	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2057	AssertRCReturn(rc, rc);
2058	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2059	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32EntryCtls, u32Val),
2060	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2061	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2062
2063	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2064	AssertRCReturn(rc, rc);
2065	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2066	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ExitCtls, u32Val),
2067	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2068	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2069
2070	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2071	AssertRCReturn(rc, rc);
2072	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2073	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32PinCtls, u32Val),
2074	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2075	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2076
2077	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2078	AssertRCReturn(rc, rc);
2079	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2080	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls, u32Val),
2081	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2082	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2083
2084	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2085	{
2086	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2087	AssertRCReturn(rc, rc);
2088	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2089	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls2, u32Val),
2090	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2091	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2092	}
2093
2094	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2095	AssertRCReturn(rc, rc);
2096	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2097	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32XcptBitmap, u32Val),
2098	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2099	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2100
2101	uint64_t u64Val;
2102	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2103	AssertRCReturn(rc, rc);
2104	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2105	("Cache=%#RX64 VMCS=%#RX64\n", pVmcsInfo->u64TscOffset, u64Val),
2106	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2107	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2108
2109	return VINF_SUCCESS;
2110	}
2111
2112
2113	#ifdef VBOX_STRICT
2114	/**
2115	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2116	*
2117	* @param pVCpu The cross context virtual CPU structure.
2118	* @param pVmcsInfo The VMCS info. object.
2119	*/
2120	static void hmR0VmxCheckHostEferMsr(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2121	{
2122	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2123
2124	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2125	{
2126	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2127	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2128	uint64_t uVmcsEferMsrVmcs;
2129	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2130	AssertRC(rc);
2131
2132	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2133	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2134	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2135	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2136	}
2137	}
2138
2139
2140	/**
2141	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2142	* VMCS are correct.
2143	*
2144	* @param pVCpu The cross context virtual CPU structure.
2145	* @param pVmcsInfo The VMCS info. object.
2146	*/
2147	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2148	{
2149	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2150
2151	/* Verify MSR counts in the VMCS are what we think it should be. */
2152	uint32_t cMsrs;
2153	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs);
2154	AssertRC(rc);
2155	Assert(cMsrs == pVmcsInfo->cEntryMsrLoad);
2156
2157	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs);
2158	AssertRC(rc);
2159	Assert(cMsrs == pVmcsInfo->cExitMsrStore);
2160
2161	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs);
2162	AssertRC(rc);
2163	Assert(cMsrs == pVmcsInfo->cExitMsrLoad);
2164
2165	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2166	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2167
2168	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2169	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2170	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2171	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2172	for (uint32_t i = 0; i < cMsrs; i++)
2173	{
2174	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2175	if (fSeparateExitMsrStorePage)
2176	{
2177	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2178	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2179	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2180	}
2181
2182	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2183	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2184	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2185
2186	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2187	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2188	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2189	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2190
2191	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2192	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2193	{
2194	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2195	if (pGuestMsrLoad->u32Msr == MSR_K6_EFER)
2196	{
2197	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2198	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2199	}
2200	else
2201	{
2202	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2203	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2204	}
2205	}
2206
2207	/* Move to the next MSR. */
2208	pHostMsrLoad++;
2209	pGuestMsrLoad++;
2210	pGuestMsrStore++;
2211	}
2212	}
2213	#endif /* VBOX_STRICT */
2214
2215
2216	/**
2217	* Flushes the TLB using EPT.
2218	*
2219	* @returns VBox status code.
2220	* @param pVCpu The cross context virtual CPU structure of the calling
2221	* EMT. Can be NULL depending on @a enmTlbFlush.
2222	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2223	* enmTlbFlush.
2224	* @param enmTlbFlush Type of flush.
2225	*
2226	* @remarks Caller is responsible for making sure this function is called only
2227	* when NestedPaging is supported and providing @a enmTlbFlush that is
2228	* supported by the CPU.
2229	* @remarks Can be called with interrupts disabled.
2230	*/
2231	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2232	{
2233	uint64_t au64Descriptor[2];
2234	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2235	au64Descriptor[0] = 0;
2236	else
2237	{
2238	Assert(pVCpu);
2239	Assert(pVmcsInfo);
2240	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2241	}
2242	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2243
2244	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2245	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2246
2247	if ( RT_SUCCESS(rc)
2248	&& pVCpu)
2249	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2250	}
2251
2252
2253	/**
2254	* Flushes the TLB using VPID.
2255	*
2256	* @returns VBox status code.
2257	* @param pVCpu The cross context virtual CPU structure of the calling
2258	* EMT. Can be NULL depending on @a enmTlbFlush.
2259	* @param enmTlbFlush Type of flush.
2260	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2261	* on @a enmTlbFlush).
2262	*
2263	* @remarks Can be called with interrupts disabled.
2264	*/
2265	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2266	{
2267	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2268
2269	uint64_t au64Descriptor[2];
2270	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2271	{
2272	au64Descriptor[0] = 0;
2273	au64Descriptor[1] = 0;
2274	}
2275	else
2276	{
2277	AssertPtr(pVCpu);
2278	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2279	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2280	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2281	au64Descriptor[1] = GCPtr;
2282	}
2283
2284	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2285	AssertMsg(rc == VINF_SUCCESS,
2286	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2287
2288	if ( RT_SUCCESS(rc)
2289	&& pVCpu)
2290	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2291	NOREF(rc);
2292	}
2293
2294
2295	/**
2296	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2297	* otherwise there is nothing really to invalidate.
2298	*
2299	* @returns VBox status code.
2300	* @param pVCpu The cross context virtual CPU structure.
2301	* @param GCVirt Guest virtual address of the page to invalidate.
2302	*/
2303	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2304	{
2305	AssertPtr(pVCpu);
2306	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2307
2308	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2309	{
2310	/*
2311	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2312	* the EPT case. See @bugref{6043} and @bugref{6177}.
2313	*
2314	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2315	* as this function maybe called in a loop with individual addresses.
2316	*/
2317	PVM pVM = pVCpu->CTX_SUFF(pVM);
2318	if (pVM->hm.s.vmx.fVpid)
2319	{
2320	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2321
2322	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2323	/*
2324	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2325	* where executing INVVPID outside 64-bit mode does not flush translations of
2326	* 64-bit linear addresses, see @bugref{6208#c72}.
2327	*/
2328	if (RT_HI_U32(GCVirt))
2329	fVpidFlush = false;
2330	#endif
2331
2332	if (fVpidFlush)
2333	{
2334	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2335	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2336	}
2337	else
2338	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2339	}
2340	else if (pVM->hm.s.fNestedPaging)
2341	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2342	}
2343
2344	return VINF_SUCCESS;
2345	}
2346
2347
2348	/**
2349	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2350	* case where neither EPT nor VPID is supported by the CPU.
2351	*
2352	* @param pHostCpu The HM physical-CPU structure.
2353	* @param pVCpu The cross context virtual CPU structure.
2354	*
2355	* @remarks Called with interrupts disabled.
2356	*/
2357	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2358	{
2359	AssertPtr(pVCpu);
2360	AssertPtr(pHostCpu);
2361
2362	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2363
2364	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2365	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2366	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2367	pVCpu->hm.s.fForceTLBFlush = false;
2368	return;
2369	}
2370
2371
2372	/**
2373	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2374	*
2375	* @param pHostCpu The HM physical-CPU structure.
2376	* @param pVCpu The cross context virtual CPU structure.
2377	* @param pVmcsInfo The VMCS info. object.
2378	*
2379	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2380	* nomenclature. The reason is, to avoid confusion in compare statements
2381	* since the host-CPU copies are named "ASID".
2382	*
2383	* @remarks Called with interrupts disabled.
2384	*/
2385	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2386	{
2387	#ifdef VBOX_WITH_STATISTICS
2388	bool fTlbFlushed = false;
2389	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2390	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2391	if (!fTlbFlushed) \
2392	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2393	} while (0)
2394	#else
2395	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2396	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2397	#endif
2398
2399	AssertPtr(pVCpu);
2400	AssertPtr(pHostCpu);
2401	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2402
2403	PVM pVM = pVCpu->CTX_SUFF(pVM);
2404	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2405	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2406	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2407
2408	/*
2409	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2410	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2411	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2412	* cannot reuse the current ASID anymore.
2413	*/
2414	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2415	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2416	{
2417	++pHostCpu->uCurrentAsid;
2418	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2419	{
2420	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2421	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2422	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2423	}
2424
2425	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2426	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2427	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2428
2429	/*
2430	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2431	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2432	*/
2433	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2434	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2435	HMVMX_SET_TAGGED_TLB_FLUSHED();
2436	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2437	}
2438	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2439	{
2440	/*
2441	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2442	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2443	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2444	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2445	* mappings, see @bugref{6568}.
2446	*
2447	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2448	*/
2449	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2450	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2451	HMVMX_SET_TAGGED_TLB_FLUSHED();
2452	}
2453
2454	pVCpu->hm.s.fForceTLBFlush = false;
2455	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2456
2457	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
2458	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
2459	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2460	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2461	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2462	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2463	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2464	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2465	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2466
2467	/* Update VMCS with the VPID. */
2468	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2469	AssertRC(rc);
2470
2471	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2472	}
2473
2474
2475	/**
2476	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2477	*
2478	* @param pHostCpu The HM physical-CPU structure.
2479	* @param pVCpu The cross context virtual CPU structure.
2480	* @param pVmcsInfo The VMCS info. object.
2481	*
2482	* @remarks Called with interrupts disabled.
2483	*/
2484	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2485	{
2486	AssertPtr(pVCpu);
2487	AssertPtr(pHostCpu);
2488	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2489	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2490	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2491
2492	/*
2493	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2494	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2495	*/
2496	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2497	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2498	{
2499	pVCpu->hm.s.fForceTLBFlush = true;
2500	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2501	}
2502
2503	/* Check for explicit TLB flushes. */
2504	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2505	{
2506	pVCpu->hm.s.fForceTLBFlush = true;
2507	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2508	}
2509
2510	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2511	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2512
2513	if (pVCpu->hm.s.fForceTLBFlush)
2514	{
2515	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2516	pVCpu->hm.s.fForceTLBFlush = false;
2517	}
2518	}
2519
2520
2521	/**
2522	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2523	*
2524	* @param pHostCpu The HM physical-CPU structure.
2525	* @param pVCpu The cross context virtual CPU structure.
2526	*
2527	* @remarks Called with interrupts disabled.
2528	*/
2529	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2530	{
2531	AssertPtr(pVCpu);
2532	AssertPtr(pHostCpu);
2533	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2534	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2535	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2536
2537	/*
2538	* Force a TLB flush for the first world switch if the current CPU differs from the one we
2539	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2540	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2541	* cannot reuse the current ASID anymore.
2542	*/
2543	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2544	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2545	{
2546	pVCpu->hm.s.fForceTLBFlush = true;
2547	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2548	}
2549
2550	/* Check for explicit TLB flushes. */
2551	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2552	{
2553	/*
2554	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2555	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2556	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
2557	* include fExplicitFlush's too) - an obscure corner case.
2558	*/
2559	pVCpu->hm.s.fForceTLBFlush = true;
2560	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2561	}
2562
2563	PVM pVM = pVCpu->CTX_SUFF(pVM);
2564	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2565	if (pVCpu->hm.s.fForceTLBFlush)
2566	{
2567	++pHostCpu->uCurrentAsid;
2568	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2569	{
2570	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2571	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2572	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2573	}
2574
2575	pVCpu->hm.s.fForceTLBFlush = false;
2576	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2577	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2578	if (pHostCpu->fFlushAsidBeforeUse)
2579	{
2580	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2581	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2582	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2583	{
2584	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2585	pHostCpu->fFlushAsidBeforeUse = false;
2586	}
2587	else
2588	{
2589	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2590	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2591	}
2592	}
2593	}
2594
2595	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2596	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2597	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2598	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2599	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2600	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2601	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2602
2603	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2604	AssertRC(rc);
2605	}
2606
2607
2608	/**
2609	* Flushes the guest TLB entry based on CPU capabilities.
2610	*
2611	* @param pHostCpu The HM physical-CPU structure.
2612	* @param pVCpu The cross context virtual CPU structure.
2613	* @param pVmcsInfo The VMCS info. object.
2614	*
2615	* @remarks Called with interrupts disabled.
2616	*/
2617	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2618	{
2619	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2620	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2621	#endif
2622	PVM pVM = pVCpu->CTX_SUFF(pVM);
2623	switch (pVM->hm.s.vmx.enmTlbFlushType)
2624	{
2625	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
2626	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
2627	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
2628	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
2629	default:
2630	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2631	break;
2632	}
2633	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2634	}
2635
2636
2637	/**
2638	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2639	* TLB entries from the host TLB before VM-entry.
2640	*
2641	* @returns VBox status code.
2642	* @param pVM The cross context VM structure.
2643	*/
2644	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2645	{
2646	/*
2647	* Determine optimal flush type for nested paging.
2648	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
2649	* unrestricted guest execution (see hmR3InitFinalizeR0()).
2650	*/
2651	if (pVM->hm.s.fNestedPaging)
2652	{
2653	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2654	{
2655	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2656	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2657	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2658	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2659	else
2660	{
2661	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2662	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2663	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2664	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2665	}
2666
2667	/* Make sure the write-back cacheable memory type for EPT is supported. */
2668	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2669	{
2670	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2671	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2672	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2673	}
2674
2675	/* EPT requires a page-walk length of 4. */
2676	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2677	{
2678	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2679	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2680	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2681	}
2682	}
2683	else
2684	{
2685	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2686	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2687	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2688	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2689	}
2690	}
2691
2692	/*
2693	* Determine optimal flush type for VPID.
2694	*/
2695	if (pVM->hm.s.vmx.fVpid)
2696	{
2697	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2698	{
2699	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2700	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2701	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2702	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2703	else
2704	{
2705	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2706	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2707	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2708	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2709	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2710	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2711	pVM->hm.s.vmx.fVpid = false;
2712	}
2713	}
2714	else
2715	{
2716	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2717	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2718	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2719	pVM->hm.s.vmx.fVpid = false;
2720	}
2721	}
2722
2723	/*
2724	* Setup the handler for flushing tagged-TLBs.
2725	*/
2726	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2727	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2728	else if (pVM->hm.s.fNestedPaging)
2729	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2730	else if (pVM->hm.s.vmx.fVpid)
2731	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2732	else
2733	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2734	return VINF_SUCCESS;
2735	}
2736
2737
2738	/**
2739	* Sets up the virtual-APIC page address for the VMCS.
2740	*
2741	* @returns VBox status code.
2742	* @param pVCpu The cross context virtual CPU structure.
2743	* @param pVmcsInfo The VMCS info. object.
2744	*/
2745	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2746	{
2747	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2748	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
2749	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
2750	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2751	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
2752	}
2753
2754
2755	/**
2756	* Sets up the MSR-bitmap address for the VMCS.
2757	*
2758	* @returns VBox status code.
2759	* @param pVCpu The cross context virtual CPU structure.
2760	* @param pVmcsInfo The VMCS info. object.
2761	*/
2762	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2763	{
2764	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2765	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
2766	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
2767	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2768	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
2769	}
2770
2771
2772	/**
2773	* Sets up the APIC-access page address for the VMCS.
2774	*
2775	* @returns VBox status code.
2776	* @param pVCpu The cross context virtual CPU structure.
2777	*/
2778	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
2779	{
2780	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
2781	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
2782	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2783	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
2784	}
2785
2786
2787	/**
2788	* Sets up the VMCS link pointer for the VMCS.
2789	*
2790	* @returns VBox status code.
2791	* @param pVCpu The cross context virtual CPU structure.
2792	* @param pVmcsInfo The VMCS info. object.
2793	*/
2794	DECLINLINE(int) hmR0VmxSetupVmcsLinkPtr(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2795	{
2796	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2797	uint64_t const u64VmcsLinkPtr = pVmcsInfo->u64VmcsLinkPtr;
2798	Assert(u64VmcsLinkPtr == UINT64_C(0xffffffffffffffff)); /* Bits 63:0 MB1. */
2799	return VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, u64VmcsLinkPtr);
2800	}
2801
2802
2803	/**
2804	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
2805	* in the VMCS.
2806	*
2807	* @returns VBox status code.
2808	* @param pVCpu The cross context virtual CPU structure.
2809	* @param pVmcsInfo The VMCS info. object.
2810	*/
2811	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2812	{
2813	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2814
2815	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
2816	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
2817	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2818
2819	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
2820	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
2821	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
2822
2823	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
2824	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
2825	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2826
2827	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
2828	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
2829	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
2830	AssertRCReturn(rc, rc);
2831	return VINF_SUCCESS;
2832	}
2833
2834
2835	/**
2836	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
2837	*
2838	* @param pVCpu The cross context virtual CPU structure.
2839	* @param pVmcsInfo The VMCS info. object.
2840	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2841	*/
2842	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2843	{
2844	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
2845
2846	/*
2847	* The guest can access the following MSRs (read, write) without causing
2848	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
2849	*/
2850	PVM pVM = pVCpu->CTX_SUFF(pVM);
2851	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
2852	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
2853	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
2854	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2855	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
2856
2857	#ifdef VBOX_STRICT
2858	/** @todo NSTVMX: Remove this later. */
2859	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_IA32_SYSENTER_CS);
2860	Assert((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR);
2861
2862	fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K8_GS_BASE);
2863	Assert((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR);
2864	#endif
2865
2866	/*
2867	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
2868	* associated with then. We never need to intercept access (writes need to be
2869	* executed without causing a VM-exit, reads will #GP fault anyway).
2870	*
2871	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
2872	* read/write them. We swap the the guest/host MSR value using the
2873	* auto-load/store MSR area.
2874	*/
2875	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2876	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
2877	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
2878	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
2879	if (pVM->cpum.ro.GuestFeatures.fIbrs)
2880	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
2881
2882	/*
2883	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
2884	*/
2885
2886	#if HC_ARCH_BITS == 64
2887	/*
2888	* Allow full read/write access for the following MSRs (mandatory for VT-x)
2889	* required for 64-bit guests.
2890	*/
2891	if (pVM->hm.s.fAllow64BitGuests)
2892	{
2893	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
2894	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
2895	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
2896	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2897
2898	# ifdef VBOX_STRICT
2899	fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K8_GS_BASE);
2900	Assert((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR);
2901	# endif
2902	}
2903	#endif
2904	}
2905
2906
2907	/**
2908	* Sets up pin-based VM-execution controls in the VMCS.
2909	*
2910	* @returns VBox status code.
2911	* @param pVCpu The cross context virtual CPU structure.
2912	* @param pVmcsInfo The VMCS info. object.
2913	*/
2914	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2915	{
2916	PVM pVM = pVCpu->CTX_SUFF(pVM);
2917	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
2918	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2919
2920	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2921	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2922
2923	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2924	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2925
2926	/* Enable the VMX-preemption timer. */
2927	if (pVM->hm.s.vmx.fUsePreemptTimer)
2928	{
2929	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2930	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
2931	}
2932
2933	#if 0
2934	/* Enable posted-interrupt processing. */
2935	if (pVM->hm.s.fPostedIntrs)
2936	{
2937	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2938	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2939	fVal \|= VMX_PIN_CTL_POSTED_INT;
2940	}
2941	#endif
2942
2943	if ((fVal & fZap) != fVal)
2944	{
2945	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2946	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
2947	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2948	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2949	}
2950
2951	/* Commit it to the VMCS and update our cache. */
2952	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2953	AssertRCReturn(rc, rc);
2954	pVmcsInfo->u32PinCtls = fVal;
2955
2956	return VINF_SUCCESS;
2957	}
2958
2959
2960	/**
2961	* Sets up secondary processor-based VM-execution controls in the VMCS.
2962	*
2963	* @returns VBox status code.
2964	* @param pVCpu The cross context virtual CPU structure.
2965	* @param pVmcsInfo The VMCS info. object.
2966	*/
2967	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2968	{
2969	PVM pVM = pVCpu->CTX_SUFF(pVM);
2970	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
2971	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2972
2973	/* WBINVD causes a VM-exit. */
2974	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2975	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
2976
2977	/* Enable EPT (aka nested-paging). */
2978	if (pVM->hm.s.fNestedPaging)
2979	fVal \|= VMX_PROC_CTLS2_EPT;
2980
2981	/* Enable the INVPCID instruction if supported by the hardware and we expose
2982	it to the guest. Without this, guest executing INVPCID would cause a #UD. */
2983	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2984	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
2985	fVal \|= VMX_PROC_CTLS2_INVPCID;
2986
2987	/* Enable VPID. */
2988	if (pVM->hm.s.vmx.fVpid)
2989	fVal \|= VMX_PROC_CTLS2_VPID;
2990
2991	/* Enable unrestricted guest execution. */
2992	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2993	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
2994
2995	#if 0
2996	if (pVM->hm.s.fVirtApicRegs)
2997	{
2998	/* Enable APIC-register virtualization. */
2999	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3000	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3001
3002	/* Enable virtual-interrupt delivery. */
3003	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3004	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3005	}
3006	#endif
3007
3008	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
3009	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3010	* done dynamically. */
3011	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3012	{
3013	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3014	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3015	AssertRCReturn(rc, rc);
3016	}
3017
3018	/* Enable the RDTSCP instruction if supported by the hardware and we expose
3019	it to the guest. Without this, guest executing RDTSCP would cause a #UD. */
3020	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
3021	&& pVM->cpum.ro.GuestFeatures.fRdTscP)
3022	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3023
3024	/* Enable Pause-Loop exiting. */
3025	if ( pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT
3026	&& pVM->hm.s.vmx.cPleGapTicks
3027	&& pVM->hm.s.vmx.cPleWindowTicks)
3028	{
3029	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3030
3031	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3032	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3033	AssertRCReturn(rc, rc);
3034	}
3035
3036	if ((fVal & fZap) != fVal)
3037	{
3038	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3039	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3040	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3041	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3042	}
3043
3044	/* Commit it to the VMCS and update our cache. */
3045	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3046	AssertRCReturn(rc, rc);
3047	pVmcsInfo->u32ProcCtls2 = fVal;
3048
3049	return VINF_SUCCESS;
3050	}
3051
3052
3053	/**
3054	* Sets up processor-based VM-execution controls in the VMCS.
3055	*
3056	* @returns VBox status code.
3057	* @param pVCpu The cross context virtual CPU structure.
3058	* @param pVmcsInfo The VMCS info. object.
3059	*/
3060	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3061	{
3062	PVM pVM = pVCpu->CTX_SUFF(pVM);
3063
3064	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3065	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3066
3067	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3068	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3069	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3070	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3071	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3072	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3073	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3074
3075	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3076	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3077	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3078	{
3079	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3080	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3081	}
3082
3083	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3084	if (!pVM->hm.s.fNestedPaging)
3085	{
3086	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3087	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3088	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3089	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3090	}
3091
3092	/* Use TPR shadowing if supported by the CPU. */
3093	if ( PDMHasApic(pVM)
3094	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3095	{
3096	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3097	/* CR8 writes cause a VM-exit based on TPR threshold. */
3098	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3099	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3100	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3101	AssertRCReturn(rc, rc);
3102	}
3103	else
3104	{
3105	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3106	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3107	if (pVM->hm.s.fAllow64BitGuests)
3108	{
3109	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3110	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3111	}
3112	}
3113
3114	/* Use MSR-bitmaps if supported by the CPU. */
3115	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3116	{
3117	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3118	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3119	AssertRCReturn(rc, rc);
3120	}
3121
3122	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3123	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3124	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3125
3126	if ((fVal & fZap) != fVal)
3127	{
3128	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3129	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3130	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3131	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3132	}
3133
3134	/* Commit it to the VMCS and update our cache. */
3135	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3136	AssertRCReturn(rc, rc);
3137	pVmcsInfo->u32ProcCtls = fVal;
3138
3139	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3140	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3141	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo, false /* fIsNstGstVmcs */);
3142
3143	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3144	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3145	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3146
3147	/* Sanity check, should not really happen. */
3148	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3149	{ /* likely */ }
3150	else
3151	{
3152	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3153	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3154	}
3155
3156	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3157	return VINF_SUCCESS;
3158	}
3159
3160
3161	/**
3162	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3163	* Processor-based VM-execution) control fields in the VMCS.
3164	*
3165	* @returns VBox status code.
3166	* @param pVCpu The cross context virtual CPU structure.
3167	* @param pVmcsInfo The VMCS info. object.
3168	*/
3169	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3170	{
3171	/* Set the auto-load/store MSR area addresses in the VMCS. */
3172	int rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3173	if (RT_SUCCESS(rc))
3174	{
3175	/* Set the VMCS link pointer in the VMCS. */
3176	rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3177	if (RT_SUCCESS(rc))
3178	{
3179	/* Set the CR0/CR4 guest/host mask. */
3180	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3181	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3182	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3183	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3184	if (RT_SUCCESS(rc))
3185	{
3186	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3187	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3188	return VINF_SUCCESS;
3189	}
3190	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3191	}
3192	else
3193	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3194	}
3195	else
3196	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3197	return rc;
3198	}
3199
3200
3201	/**
3202	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3203	*
3204	* We shall setup those exception intercepts that don't change during the
3205	* lifetime of the VM here. The rest are done dynamically while loading the
3206	* guest state.
3207	*
3208	* @returns VBox status code.
3209	* @param pVCpu The cross context virtual CPU structure.
3210	* @param pVmcsInfo The VMCS info. object.
3211	*/
3212	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3213	{
3214	/*
3215	* The following exceptions are always intercepted:
3216	*
3217	* #AC - To prevent the guest from hanging the CPU.
3218	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3219	* recursive #DBs can cause a CPU hang.
3220	* #PF - To sync our shadow page tables when nested-paging is not used.
3221	*/
3222	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3223	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3224	\| RT_BIT(X86_XCPT_DB)
3225	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3226
3227	/* Commit it to the VMCS. */
3228	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3229	AssertRCReturn(rc, rc);
3230
3231	/* Update our cache of the exception bitmap. */
3232	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3233	return VINF_SUCCESS;
3234	}
3235
3236
3237	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3238	/**
3239	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3240	*
3241	* @returns VBox status code.
3242	* @param pVCpu The cross context virtual CPU structure.
3243	* @param pVmcsInfo The VMCS info. object.
3244	*/
3245	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3246	{
3247	PVM pVM = pVCpu->CTX_SUFF(pVM);
3248	int rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3249	if (RT_SUCCESS(rc))
3250	{
3251	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3252	if (RT_SUCCESS(rc))
3253	{
3254	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3255	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3256	if (RT_SUCCESS(rc))
3257	{
3258	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3259	rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3260	if (RT_SUCCESS(rc))
3261	return VINF_SUCCESS;
3262
3263	LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3264	}
3265	else
3266	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3267	}
3268	else
3269	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3270	}
3271	else
3272	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3273
3274	return rc;
3275	}
3276	#endif
3277
3278
3279	/**
3280	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3281	* VMX.
3282	*
3283	* @returns VBox status code.
3284	* @param pVCpu The cross context virtual CPU structure.
3285	* @param pVmcsInfo The VMCS info. object.
3286	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3287	*/
3288	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3289	{
3290	Assert(pVmcsInfo);
3291	Assert(pVmcsInfo->pvVmcs);
3292	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3293
3294	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3295	PVM pVM = pVCpu->CTX_SUFF(pVM);
3296	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3297	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3298
3299	LogFlowFunc(("\n"));
3300
3301	/*
3302	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3303	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3304	*/
3305	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3306	if (RT_SUCCESS(rc))
3307	{
3308	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3309	if (RT_SUCCESS(rc))
3310	{
3311	if (!fIsNstGstVmcs)
3312	{
3313	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3314	if (RT_SUCCESS(rc))
3315	{
3316	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
3317	if (RT_SUCCESS(rc))
3318	{
3319	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
3320	if (RT_SUCCESS(rc))
3321	{
3322	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
3323	if (RT_SUCCESS(rc))
3324	{ /* likely */ }
3325	else
3326	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
3327	}
3328	else
3329	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
3330	}
3331	else
3332	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
3333	}
3334	else
3335	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
3336	}
3337	else
3338	{
3339	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3340	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
3341	if (RT_SUCCESS(rc))
3342	{ /* likely */ }
3343	else
3344	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
3345	#else
3346	AssertFailed();
3347	#endif
3348	}
3349	}
3350	else
3351	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
3352	}
3353	else
3354	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
3355
3356	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
3357	if (RT_SUCCESS(rc))
3358	{
3359	rc = hmR0VmxClearVmcs(pVmcsInfo);
3360	if (RT_SUCCESS(rc))
3361	{ /* likely */ }
3362	else
3363	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
3364	}
3365
3366	/*
3367	* Update the last-error record both for failures and success, so we
3368	* can propagate the status code back to ring-3 for diagnostics.
3369	*/
3370	hmR0VmxUpdateErrorRecord(pVCpu, rc);
3371	NOREF(pszVmcs);
3372	return rc;
3373	}
3374
3375
3376	/**
3377	* Does global VT-x initialization (called during module initialization).
3378	*
3379	* @returns VBox status code.
3380	*/
3381	VMMR0DECL(int) VMXR0GlobalInit(void)
3382	{
3383	#ifdef HMVMX_USE_FUNCTION_TABLE
3384	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
3385	# ifdef VBOX_STRICT
3386	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
3387	Assert(g_apfnVMExitHandlers[i]);
3388	# endif
3389	#endif
3390	return VINF_SUCCESS;
3391	}
3392
3393
3394	/**
3395	* Does global VT-x termination (called during module termination).
3396	*/
3397	VMMR0DECL(void) VMXR0GlobalTerm()
3398	{
3399	/* Nothing to do currently. */
3400	}
3401
3402
3403	/**
3404	* Sets up and activates VT-x on the current CPU.
3405	*
3406	* @returns VBox status code.
3407	* @param pHostCpu The HM physical-CPU structure.
3408	* @param pVM The cross context VM structure. Can be
3409	* NULL after a host resume operation.
3410	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
3411	* fEnabledByHost is @c true).
3412	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
3413	* @a fEnabledByHost is @c true).
3414	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
3415	* enable VT-x on the host.
3416	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
3417	*/
3418	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
3419	PCSUPHWVIRTMSRS pHwvirtMsrs)
3420	{
3421	Assert(pHostCpu);
3422	Assert(pHwvirtMsrs);
3423	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3424
3425	/* Enable VT-x if it's not already enabled by the host. */
3426	if (!fEnabledByHost)
3427	{
3428	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
3429	if (RT_FAILURE(rc))
3430	return rc;
3431	}
3432
3433	/*
3434	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
3435	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
3436	* invalidated when flushing by VPID.
3437	*/
3438	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3439	{
3440	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
3441	pHostCpu->fFlushAsidBeforeUse = false;
3442	}
3443	else
3444	pHostCpu->fFlushAsidBeforeUse = true;
3445
3446	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
3447	++pHostCpu->cTlbFlushes;
3448
3449	return VINF_SUCCESS;
3450	}
3451
3452
3453	/**
3454	* Deactivates VT-x on the current CPU.
3455	*
3456	* @returns VBox status code.
3457	* @param pvCpuPage Pointer to the VMXON region.
3458	* @param HCPhysCpuPage Physical address of the VMXON region.
3459	*
3460	* @remarks This function should never be called when SUPR0EnableVTx() or
3461	* similar was used to enable VT-x on the host.
3462	*/
3463	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
3464	{
3465	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
3466
3467	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3468	return hmR0VmxLeaveRootMode();
3469	}
3470
3471
3472	/**
3473	* Does per-VM VT-x initialization.
3474	*
3475	* @returns VBox status code.
3476	* @param pVM The cross context VM structure.
3477	*/
3478	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
3479	{
3480	LogFlowFunc(("pVM=%p\n", pVM));
3481
3482	int rc = hmR0VmxStructsAlloc(pVM);
3483	if (RT_FAILURE(rc))
3484	{
3485	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
3486	return rc;
3487	}
3488
3489	return VINF_SUCCESS;
3490	}
3491
3492
3493	/**
3494	* Does per-VM VT-x termination.
3495	*
3496	* @returns VBox status code.
3497	* @param pVM The cross context VM structure.
3498	*/
3499	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
3500	{
3501	LogFlowFunc(("pVM=%p\n", pVM));
3502
3503	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3504	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
3505	{
3506	Assert(pVM->hm.s.vmx.pvScratch);
3507	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
3508	}
3509	#endif
3510	hmR0VmxStructsFree(pVM);
3511	return VINF_SUCCESS;
3512	}
3513
3514
3515	/**
3516	* Sets up the VM for execution using hardware-assisted VMX.
3517	* This function is only called once per-VM during initialization.
3518	*
3519	* @returns VBox status code.
3520	* @param pVM The cross context VM structure.
3521	*/
3522	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
3523	{
3524	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
3525	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3526
3527	LogFlowFunc(("pVM=%p\n", pVM));
3528
3529	/*
3530	* At least verify if VMX is enabled, since we can't check if we're in
3531	* VMX root mode or not without causing a #GP.
3532	*/
3533	RTCCUINTREG const uHostCR4 = ASMGetCR4();
3534	if (RT_LIKELY(uHostCR4 & X86_CR4_VMXE))
3535	{ /* likely */ }
3536	else
3537	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
3538
3539	/*
3540	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
3541	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
3542	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
3543	*/
3544	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3545	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
3546	\|\| !pVM->hm.s.vmx.pRealModeTSS))
3547	{
3548	LogRelFunc(("Invalid real-on-v86 state.\n"));
3549	return VERR_INTERNAL_ERROR;
3550	}
3551
3552	/* Initialize these always, see hmR3InitFinalizeR0().*/
3553	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
3554	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
3555
3556	/* Setup the tagged-TLB flush handlers. */
3557	int rc = hmR0VmxSetupTaggedTlb(pVM);
3558	if (RT_FAILURE(rc))
3559	{
3560	LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
3561	return rc;
3562	}
3563
3564	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
3565	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
3566	#if HC_ARCH_BITS == 64
3567	if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
3568	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
3569	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
3570	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
3571	#endif
3572
3573	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3574	{
3575	PVMCPU pVCpu = &pVM->aCpus[idCpu];
3576	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
3577
3578	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
3579	if (RT_SUCCESS(rc))
3580	{
3581	#if HC_ARCH_BITS == 32
3582	hmR0VmxInitVmcsReadCache(pVCpu);
3583	#endif
3584	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3585	if (pVM->cpum.ro.GuestFeatures.fVmx)
3586	{
3587	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
3588	if (RT_SUCCESS(rc))
3589	{ /* likely */ }
3590	else
3591	{
3592	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
3593	return rc;
3594	}
3595	}
3596	#endif
3597	}
3598	else
3599	{
3600	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
3601	return rc;
3602	}
3603	}
3604
3605	return VINF_SUCCESS;
3606	}
3607
3608
3609	#if HC_ARCH_BITS == 32
3610	# ifdef VBOX_ENABLE_64_BITS_GUESTS
3611	/**
3612	* Check if guest state allows safe use of 32-bit switcher again.
3613	*
3614	* Segment bases and protected mode structures must be 32-bit addressable
3615	* because the 32-bit switcher will ignore high dword when writing these VMCS
3616	* fields. See @bugref{8432} for details.
3617	*
3618	* @returns true if safe, false if must continue to use the 64-bit switcher.
3619	* @param pCtx Pointer to the guest-CPU context.
3620	*
3621	* @remarks No-long-jump zone!!!
3622	*/
3623	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
3624	{
3625	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
3626	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
3627	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3628	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3629	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
3630	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3631	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
3632	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
3633	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3634	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3635
3636	/* All good, bases are 32-bit. */
3637	return true;
3638	}
3639	# endif /* VBOX_ENABLE_64_BITS_GUESTS */
3640
3641	# ifdef VBOX_STRICT
3642	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
3643	{
3644	switch (idxField)
3645	{
3646	case VMX_VMCS_GUEST_RIP:
3647	case VMX_VMCS_GUEST_RSP:
3648	case VMX_VMCS_GUEST_SYSENTER_EIP:
3649	case VMX_VMCS_GUEST_SYSENTER_ESP:
3650	case VMX_VMCS_GUEST_GDTR_BASE:
3651	case VMX_VMCS_GUEST_IDTR_BASE:
3652	case VMX_VMCS_GUEST_CS_BASE:
3653	case VMX_VMCS_GUEST_DS_BASE:
3654	case VMX_VMCS_GUEST_ES_BASE:
3655	case VMX_VMCS_GUEST_FS_BASE:
3656	case VMX_VMCS_GUEST_GS_BASE:
3657	case VMX_VMCS_GUEST_SS_BASE:
3658	case VMX_VMCS_GUEST_LDTR_BASE:
3659	case VMX_VMCS_GUEST_TR_BASE:
3660	case VMX_VMCS_GUEST_CR3:
3661	return true;
3662	}
3663	return false;
3664	}
3665
3666	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3667	{
3668	switch (idxField)
3669	{
3670	/* Read-only fields. */
3671	case VMX_VMCS_RO_EXIT_QUALIFICATION:
3672	return true;
3673	}
3674	/* Remaining readable fields should also be writable. */
3675	return hmR0VmxIsValidWriteField(idxField);
3676	}
3677	# endif /* VBOX_STRICT */
3678
3679
3680	/**
3681	* Executes the specified handler in 64-bit mode.
3682	*
3683	* @returns VBox status code (no informational status codes).
3684	* @param pVCpu The cross context virtual CPU structure.
3685	* @param enmOp The operation to perform.
3686	* @param cParams Number of parameters.
3687	* @param paParam Array of 32-bit parameters.
3688	*/
3689	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
3690	{
3691	PVM pVM = pVCpu->CTX_SUFF(pVM);
3692	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
3693	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
3694	Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
3695	Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
3696
3697	#ifdef VBOX_STRICT
3698	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
3699	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
3700
3701	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
3702	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
3703	#endif
3704
3705	/* Disable interrupts. */
3706	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
3707
3708	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
3709	RTCPUID idHostCpu = RTMpCpuId();
3710	CPUMR0SetLApic(pVCpu, idHostCpu);
3711	#endif
3712
3713	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
3714
3715	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3716	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3717
3718	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
3719	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3720	hmR0VmxClearVmcs(pVmcsInfo);
3721
3722	/* Leave VMX root mode and disable VMX. */
3723	VMXDisable();
3724	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3725
3726	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
3727	CPUMSetHyperEIP(pVCpu, enmOp);
3728	for (int i = (int)cParams - 1; i >= 0; i--)
3729	CPUMPushHyper(pVCpu, paParam[i]);
3730
3731	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
3732
3733	/* Call the switcher. */
3734	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
3735	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
3736
3737	/* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
3738	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
3739
3740	/* Re-enter VMX root mode. */
3741	int rc2 = VMXEnable(HCPhysCpuPage);
3742	if (RT_FAILURE(rc2))
3743	{
3744	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3745	ASMSetFlags(fOldEFlags);
3746	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
3747	return rc2;
3748	}
3749
3750	/* Restore the VMCS as the current VMCS. */
3751	rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
3752	AssertRC(rc2);
3753	Assert(!(ASMGetFlags() & X86_EFL_IF));
3754	ASMSetFlags(fOldEFlags);
3755	return rc;
3756	}
3757
3758
3759	/**
3760	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
3761	* supporting 64-bit guests.
3762	*
3763	* @returns VBox status code.
3764	* @param fResume Whether to VMLAUNCH or VMRESUME.
3765	* @param pCtx Pointer to the guest-CPU context.
3766	* @param pCache Pointer to the VMCS batch cache.
3767	* @param pVM The cross context VM structure.
3768	* @param pVCpu The cross context virtual CPU structure.
3769	*/
3770	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
3771	{
3772	NOREF(fResume);
3773
3774	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3775	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3776	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3777
3778	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3779	pCache->uPos = 1;
3780	pCache->interPD = PGMGetInterPaeCR3(pVM);
3781	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
3782	#endif
3783
3784	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3785	pCache->TestIn.HCPhysCpuPage = 0;
3786	pCache->TestIn.HCPhysVmcs = 0;
3787	pCache->TestIn.pCache = 0;
3788	pCache->TestOut.HCPhysVmcs = 0;
3789	pCache->TestOut.pCache = 0;
3790	pCache->TestOut.pCtx = 0;
3791	pCache->TestOut.eflags = 0;
3792	#else
3793	NOREF(pCache);
3794	#endif
3795
3796	uint32_t aParam[10];
3797	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
3798	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
3799	aParam[2] = RT_LO_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
3800	aParam[3] = RT_HI_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
3801	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
3802	aParam[5] = 0;
3803	aParam[6] = VM_RC_ADDR(pVM, pVM);
3804	aParam[7] = 0;
3805	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
3806	aParam[9] = 0;
3807
3808	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3809	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
3810	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
3811	#endif
3812	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
3813
3814	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3815	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
3816	Assert(pCtx->dr[4] == 10);
3817	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
3818	#endif
3819
3820	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3821	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
3822	AssertMsg(pCache->TestIn.HCPhysVmcs == pVmcsInfo->HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3823	pVmcsInfo->HCPhysVmcs));
3824	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3825	pCache->TestOut.HCPhysVmcs));
3826	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
3827	pCache->TestOut.pCache));
3828	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
3829	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
3830	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
3831	pCache->TestOut.pCtx));
3832	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
3833	#endif
3834	NOREF(pCtx);
3835	return rc;
3836	}
3837	#endif
3838
3839
3840	/**
3841	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
3842	* the VMCS.
3843	*
3844	* @returns VBox status code.
3845	*/
3846	static int hmR0VmxExportHostControlRegs(void)
3847	{
3848	RTCCUINTREG uReg = ASMGetCR0();
3849	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
3850	AssertRCReturn(rc, rc);
3851
3852	uReg = ASMGetCR3();
3853	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
3854	AssertRCReturn(rc, rc);
3855
3856	uReg = ASMGetCR4();
3857	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
3858	AssertRCReturn(rc, rc);
3859	return rc;
3860	}
3861
3862
3863	/**
3864	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
3865	* the host-state area in the VMCS.
3866	*
3867	* @returns VBox status code.
3868	* @param pVCpu The cross context virtual CPU structure.
3869	*/
3870	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
3871	{
3872	#if HC_ARCH_BITS == 64
3873	/**
3874	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
3875	* requirements. See hmR0VmxExportHostSegmentRegs().
3876	*/
3877	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
3878	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
3879	{ \
3880	bool fValidSelector = true; \
3881	if ((selValue) & X86_SEL_LDT) \
3882	{ \
3883	uint32_t uAttr = ASMGetSegAttr((selValue)); \
3884	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
3885	} \
3886	if (fValidSelector) \
3887	{ \
3888	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
3889	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
3890	} \
3891	(selValue) = 0; \
3892	}
3893
3894	/*
3895	* If we've executed guest code using hardware-assisted VMX, the host-state bits
3896	* will be messed up. We should -not- save the messed up state without restoring
3897	* the original host-state, see @bugref{7240}.
3898	*
3899	* This apparently can happen (most likely the FPU changes), deal with it rather than
3900	* asserting. Was observed booting Solaris 10u10 32-bit guest.
3901	*/
3902	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
3903	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
3904	{
3905	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
3906	pVCpu->idCpu));
3907	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
3908	}
3909	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
3910	#else
3911	RT_NOREF(pVCpu);
3912	#endif
3913
3914	/*
3915	* Host DS, ES, FS and GS segment registers.
3916	*/
3917	#if HC_ARCH_BITS == 64
3918	RTSEL uSelDS = ASMGetDS();
3919	RTSEL uSelES = ASMGetES();
3920	RTSEL uSelFS = ASMGetFS();
3921	RTSEL uSelGS = ASMGetGS();
3922	#else
3923	RTSEL uSelDS = 0;
3924	RTSEL uSelES = 0;
3925	RTSEL uSelFS = 0;
3926	RTSEL uSelGS = 0;
3927	#endif
3928
3929	/*
3930	* Host CS and SS segment registers.
3931	*/
3932	RTSEL uSelCS = ASMGetCS();
3933	RTSEL uSelSS = ASMGetSS();
3934
3935	/*
3936	* Host TR segment register.
3937	*/
3938	RTSEL uSelTR = ASMGetTR();
3939
3940	#if HC_ARCH_BITS == 64
3941	/*
3942	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
3943	* gain VM-entry and restore them before we get preempted.
3944	*
3945	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
3946	*/
3947	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3948	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3949	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3950	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3951	# undef VMXLOCAL_ADJUST_HOST_SEG
3952	#endif
3953
3954	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
3955	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
3956	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
3957	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
3958	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
3959	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
3960	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
3961	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
3962	Assert(uSelCS);
3963	Assert(uSelTR);
3964
3965	/* Write these host selector fields into the host-state area in the VMCS. */
3966	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3967	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3968	#if HC_ARCH_BITS == 64
3969	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3970	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3971	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3972	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3973	#else
3974	NOREF(uSelDS);
3975	NOREF(uSelES);
3976	NOREF(uSelFS);
3977	NOREF(uSelGS);
3978	#endif
3979	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3980	AssertRCReturn(rc, rc);
3981
3982	/*
3983	* Host GDTR and IDTR.
3984	*/
3985	RTGDTR Gdtr;
3986	RTIDTR Idtr;
3987	RT_ZERO(Gdtr);
3988	RT_ZERO(Idtr);
3989	ASMGetGDTR(&Gdtr);
3990	ASMGetIDTR(&Idtr);
3991	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3992	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3993	AssertRCReturn(rc, rc);
3994
3995	#if HC_ARCH_BITS == 64
3996	/*
3997	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
3998	* them to the maximum limit (0xffff) on every VM-exit.
3999	*/
4000	if (Gdtr.cbGdt != 0xffff)
4001	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4002
4003	/*
4004	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4005	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4006	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4007	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4008	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4009	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4010	* at 0xffff on hosts where we are sure it won't cause trouble.
4011	*/
4012	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4013	if (Idtr.cbIdt < 0x0fff)
4014	# else
4015	if (Idtr.cbIdt != 0xffff)
4016	# endif
4017	{
4018	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4019	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4020	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4021	}
4022	#endif
4023
4024	/*
4025	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4026	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4027	* RPL should be too in most cases.
4028	*/
4029	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4030	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4031
4032	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4033	#if HC_ARCH_BITS == 64
4034	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4035
4036	/*
4037	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4038	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4039	* restoration if the host has something else. Task switching is not supported in 64-bit
4040	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4041	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4042	*
4043	* [1] See Intel spec. 3.5 "System Descriptor Types".
4044	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4045	*/
4046	PVM pVM = pVCpu->CTX_SUFF(pVM);
4047	Assert(pDesc->System.u4Type == 11);
4048	if ( pDesc->System.u16LimitLow != 0x67
4049	\|\| pDesc->System.u4LimitHigh)
4050	{
4051	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4052	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4053	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4054	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4055	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4056	}
4057
4058	/*
4059	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4060	*/
4061	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4062	{
4063	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4064	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4065	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4066	{
4067	/* The GDT is read-only but the writable GDT is available. */
4068	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4069	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4070	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4071	AssertRCReturn(rc, rc);
4072	}
4073	}
4074	#else
4075	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4076	#endif
4077	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4078	AssertRCReturn(rc, rc);
4079
4080	/*
4081	* Host FS base and GS base.
4082	*/
4083	#if HC_ARCH_BITS == 64
4084	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4085	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4086	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4087	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4088	AssertRCReturn(rc, rc);
4089
4090	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4091	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4092	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4093	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4094	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4095	#endif
4096	return VINF_SUCCESS;
4097	}
4098
4099
4100	/**
4101	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4102	* host-state area of the VMCS.
4103	*
4104	* These MSRs will be automatically restored on the host after every successful
4105	* VM-exit.
4106	*
4107	* @returns VBox status code.
4108	* @param pVCpu The cross context virtual CPU structure.
4109	*
4110	* @remarks No-long-jump zone!!!
4111	*/
4112	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4113	{
4114	AssertPtr(pVCpu);
4115
4116	/*
4117	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4118	* rather than swapping them on every VM-entry.
4119	*/
4120	hmR0VmxLazySaveHostMsrs(pVCpu);
4121
4122	/*
4123	* Host Sysenter MSRs.
4124	*/
4125	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4126	#if HC_ARCH_BITS == 32
4127	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4128	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4129	#else
4130	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4131	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4132	#endif
4133	AssertRCReturn(rc, rc);
4134
4135	/*
4136	* Host EFER MSR.
4137	*
4138	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4139	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4140	*/
4141	PVM pVM = pVCpu->CTX_SUFF(pVM);
4142	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4143	{
4144	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4145	AssertRCReturn(rc, rc);
4146	}
4147
4148	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4149	* hmR0VmxExportGuestEntryExitCtls(). */
4150
4151	return VINF_SUCCESS;
4152	}
4153
4154
4155	/**
4156	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4157	*
4158	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4159	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4160	*
4161	* @returns true if we need to load guest EFER, false otherwise.
4162	* @param pVCpu The cross context virtual CPU structure.
4163	*
4164	* @remarks Requires EFER, CR4.
4165	* @remarks No-long-jump zone!!!
4166	*/
4167	static bool hmR0VmxShouldSwapEferMsr(PCVMCPU pVCpu)
4168	{
4169	#ifdef HMVMX_ALWAYS_SWAP_EFER
4170	RT_NOREF(pVCpu);
4171	return true;
4172	#else
4173	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4174	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4175	/* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4176	if (CPUMIsGuestInLongModeEx(pCtx))
4177	return false;
4178	#endif
4179
4180	PVM pVM = pVCpu->CTX_SUFF(pVM);
4181	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4182	uint64_t const u64GuestEfer = pCtx->msrEFER;
4183
4184	/*
4185	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4186	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4187	*/
4188	if ( CPUMIsGuestInLongModeEx(pCtx)
4189	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4190	return true;
4191
4192	/*
4193	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4194	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4195	*
4196	* See Intel spec. 4.5 "IA-32e Paging".
4197	* See Intel spec. 4.1.1 "Three Paging Modes".
4198	*
4199	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4200	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4201	*/
4202	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4203	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4204	if ( (pCtx->cr4 & X86_CR4_PAE)
4205	&& (pCtx->cr0 & X86_CR0_PG)
4206	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4207	{
4208	/* Assert that host is NX capable. */
4209	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4210	return true;
4211	}
4212
4213	return false;
4214	#endif
4215	}
4216
4217	/**
4218	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4219	* VMCS.
4220	*
4221	* This is typically required when the guest changes paging mode.
4222	*
4223	* @returns VBox status code.
4224	* @param pVCpu The cross context virtual CPU structure.
4225	* @param pVmxTransient The VMX-transient structure.
4226	*
4227	* @remarks Requires EFER.
4228	* @remarks No-long-jump zone!!!
4229	*/
4230	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4231	{
4232	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4233	{
4234	PVM pVM = pVCpu->CTX_SUFF(pVM);
4235	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4236
4237	/*
4238	* VM-entry controls.
4239	*/
4240	{
4241	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4242	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4243
4244	/*
4245	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4246	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4247	*
4248	* For nested-guests, this is a mandatory VM-entry control. It's also
4249	* required because we do not want to leak host bits to the nested-guest.
4250	*/
4251	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4252
4253	/*
4254	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4255	*
4256	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4257	* required to get the nested-guest working with hardware-assisted VMX execution.
4258	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested-hypervisor
4259	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4260	* here rather than while merging the guest VMCS controls.
4261	*/
4262	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4263	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4264	else
4265	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4266
4267	/*
4268	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4269	*
4270	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4271	* regardless of whether the nested-guest VMCS specifies it because we are free to
4272	* load whatever MSRs we require and we do not need to modify the guest visible copy
4273	* of the VM-entry MSR load area.
4274	*/
4275	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4276	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4277	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4278	else
4279	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4280
4281	/*
4282	* The following should -not- be set (since we're not in SMM mode):
4283	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4284	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4285	*/
4286
4287	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4288	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4289
4290	if ((fVal & fZap) == fVal)
4291	{ /* likely */ }
4292	else
4293	{
4294	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4295	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4296	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4297	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4298	}
4299
4300	/* Commit it to the VMCS. */
4301	if (pVmcsInfo->u32EntryCtls != fVal)
4302	{
4303	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4304	AssertRCReturn(rc, rc);
4305	pVmcsInfo->u32EntryCtls = fVal;
4306	}
4307	}
4308
4309	/*
4310	* VM-exit controls.
4311	*/
4312	{
4313	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4314	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4315
4316	/*
4317	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4318	* supported the 1-setting of this bit.
4319	*
4320	* For nested-guests, we set the "save debug controls" as the converse
4321	* "load debug controls" is mandatory for nested-guests anyway.
4322	*/
4323	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4324
4325	/*
4326	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4327	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4328	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4329	* hmR0VmxExportHostMsrs().
4330	*
4331	* For nested-guests, we always set this bit as we do not support 32-bit
4332	* hosts.
4333	*/
4334	#if HC_ARCH_BITS == 64
4335	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4336	#else
4337	Assert(!pVmxTransient->fIsNestedGuest);
4338	Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
4339	\|\| pVmcsInfo->pfnStartVM == VMXR0StartVM32);
4340	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
4341	if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
4342	{
4343	/* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
4344	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4345	}
4346	else
4347	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
4348	#endif
4349
4350	/*
4351	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4352	*
4353	* For nested-guests, we should use the "save IA32_EFER" control if we also
4354	* used the "load IA32_EFER" control while exporting VM-entry controls.
4355	*/
4356	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4357	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4358	{
4359	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4360	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4361	}
4362
4363	/*
4364	* Enable saving of the VMX-preemption timer value on VM-exit.
4365	* For nested-guests, currently not exposed/used.
4366	*/
4367	if ( pVM->hm.s.vmx.fUsePreemptTimer
4368	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4369	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4370
4371	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4372	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4373
4374	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4375	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4376	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4377
4378	if ((fVal & fZap) == fVal)
4379	{ /* likely */ }
4380	else
4381	{
4382	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4383	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4384	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4385	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4386	}
4387
4388	/* Commit it to the VMCS. */
4389	if (pVmcsInfo->u32ExitCtls != fVal)
4390	{
4391	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4392	AssertRCReturn(rc, rc);
4393	pVmcsInfo->u32ExitCtls = fVal;
4394	}
4395	}
4396
4397	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4398	}
4399	return VINF_SUCCESS;
4400	}
4401
4402
4403	/**
4404	* Sets the TPR threshold in the VMCS.
4405	*
4406	* @returns VBox status code.
4407	* @param pVCpu The cross context virtual CPU structure.
4408	* @param pVmcsInfo The VMCS info. object.
4409	* @param u32TprThreshold The TPR threshold (task-priority class only).
4410	*/
4411	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4412	{
4413	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4414	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4415	RT_NOREF2(pVCpu, pVmcsInfo);
4416	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4417	}
4418
4419
4420	/**
4421	* Exports the guest APIC TPR state into the VMCS.
4422	*
4423	* @returns VBox status code.
4424	* @param pVCpu The cross context virtual CPU structure.
4425	* @param pVmxTransient The VMX-transient structure.
4426	*
4427	* @remarks No-long-jump zone!!!
4428	*/
4429	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4430	{
4431	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4432	{
4433	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4434
4435	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4436	if (!pVmxTransient->fIsNestedGuest)
4437	{
4438	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4439	&& APICIsEnabled(pVCpu))
4440	{
4441	/*
4442	* Setup TPR shadowing.
4443	*/
4444	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4445	{
4446	bool fPendingIntr = false;
4447	uint8_t u8Tpr = 0;
4448	uint8_t u8PendingIntr = 0;
4449	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4450	AssertRCReturn(rc, rc);
4451
4452	/*
4453	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4454	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4455	* priority of the pending interrupt so we can deliver the interrupt. If there
4456	* are no interrupts pending, set threshold to 0 to not cause any
4457	* TPR-below-threshold VM-exits.
4458	*/
4459	Assert(pVmcsInfo->pbVirtApic);
4460	pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
4461	uint32_t u32TprThreshold = 0;
4462	if (fPendingIntr)
4463	{
4464	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4465	(which is the Task-Priority Class). */
4466	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4467	const uint8_t u8TprPriority = u8Tpr >> 4;
4468	if (u8PendingPriority <= u8TprPriority)
4469	u32TprThreshold = u8PendingPriority;
4470	}
4471
4472	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
4473	AssertRCReturn(rc, rc);
4474	}
4475	}
4476	}
4477	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4478	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4479	}
4480	return VINF_SUCCESS;
4481	}
4482
4483
4484	/**
4485	* Gets the guest interruptibility-state.
4486	*
4487	* @returns Guest's interruptibility-state.
4488	* @param pVCpu The cross context virtual CPU structure.
4489	* @param pVmcsInfo The VMCS info. object.
4490	*
4491	* @remarks No-long-jump zone!!!
4492	*/
4493	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
4494	{
4495	/*
4496	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4497	*/
4498	uint32_t fIntrState = 0;
4499	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4500	{
4501	/* If inhibition is active, RIP and RFLAGS should've been updated
4502	(i.e. read previously from the VMCS or from ring-3). */
4503	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4504	#ifdef VBOX_STRICT
4505	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
4506	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4507	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
4508	#endif
4509	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
4510	{
4511	if (pCtx->eflags.Bits.u1IF)
4512	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4513	else
4514	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
4515	}
4516	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4517	{
4518	/*
4519	* We can clear the inhibit force flag as even if we go back to the recompiler
4520	* without executing guest code in VT-x, the flag's condition to be cleared is
4521	* met and thus the cleared state is correct.
4522	*/
4523	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4524	}
4525	}
4526
4527	/*
4528	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
4529	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
4530	* setting this would block host-NMIs and IRET will not clear the blocking.
4531	*
4532	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
4533	*
4534	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
4535	*/
4536	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
4537	&& CPUMIsGuestNmiBlocking(pVCpu))
4538	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
4539
4540	return fIntrState;
4541	}
4542
4543
4544	/**
4545	* Exports the exception intercepts required for guest execution in the VMCS.
4546	*
4547	* @returns VBox status code.
4548	* @param pVCpu The cross context virtual CPU structure.
4549	* @param pVmxTransient The VMX-transient structure.
4550	*
4551	* @remarks No-long-jump zone!!!
4552	*/
4553	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4554	{
4555	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
4556	{
4557	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
4558	if ( !pVmxTransient->fIsNestedGuest
4559	&& pVCpu->hm.s.fGIMTrapXcptUD)
4560	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
4561	else
4562	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
4563
4564	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
4565	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
4566	}
4567	return VINF_SUCCESS;
4568	}
4569
4570
4571	/**
4572	* Exports the guest's RIP into the guest-state area in the VMCS.
4573	*
4574	* @returns VBox status code.
4575	* @param pVCpu The cross context virtual CPU structure.
4576	*
4577	* @remarks No-long-jump zone!!!
4578	*/
4579	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
4580	{
4581	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4582	{
4583	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
4584
4585	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
4586	AssertRCReturn(rc, rc);
4587
4588	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
4589	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
4590	}
4591	return VINF_SUCCESS;
4592	}
4593
4594
4595	/**
4596	* Exports the guest's RSP into the guest-state area in the VMCS.
4597	*
4598	* @returns VBox status code.
4599	* @param pVCpu The cross context virtual CPU structure.
4600	*
4601	* @remarks No-long-jump zone!!!
4602	*/
4603	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
4604	{
4605	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
4606	{
4607	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
4608
4609	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
4610	AssertRCReturn(rc, rc);
4611
4612	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
4613	}
4614	return VINF_SUCCESS;
4615	}
4616
4617
4618	/**
4619	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
4620	*
4621	* @returns VBox status code.
4622	* @param pVCpu The cross context virtual CPU structure.
4623	* @param pVmxTransient The VMX-transient structure.
4624	*
4625	* @remarks No-long-jump zone!!!
4626	*/
4627	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4628	{
4629	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
4630	{
4631	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
4632
4633	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
4634	Let us assert it as such and use 32-bit VMWRITE. */
4635	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
4636	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
4637	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
4638	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
4639
4640	/*
4641	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
4642	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
4643	* can run the real-mode guest code under Virtual 8086 mode.
4644	*/
4645	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4646	if (pVmcsInfo->RealMode.fRealOnV86Active)
4647	{
4648	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4649	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4650	Assert(!pVmxTransient->fIsNestedGuest);
4651	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
4652	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
4653	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
4654	}
4655
4656	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
4657	AssertRCReturn(rc, rc);
4658
4659	/*
4660	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
4661	*
4662	* We must avoid setting any automatic debug exceptions delivery when single-stepping
4663	* through the hypervisor debugger using EFLAGS.TF.
4664	*/
4665	if ( !pVmxTransient->fIsNestedGuest
4666	&& !pVCpu->hm.s.fSingleInstruction
4667	&& fEFlags.Bits.u1TF)
4668	{
4669	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
4670	* premature trips to ring-3 esp since IEM does not yet handle it. */
4671	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
4672	AssertRCReturn(rc, rc);
4673	}
4674	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
4675	* nested-guest VMCS. */
4676
4677	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
4678	Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
4679	}
4680	return VINF_SUCCESS;
4681	}
4682
4683
4684	/**
4685	* Exports the guest CR0 control register into the guest-state area in the VMCS.
4686	*
4687	* The guest FPU state is always pre-loaded hence we don't need to bother about
4688	* sharing FPU related CR0 bits between the guest and host.
4689	*
4690	* @returns VBox status code.
4691	* @param pVCpu The cross context virtual CPU structure.
4692	* @param pVmxTransient The VMX-transient structure.
4693	*
4694	* @remarks No-long-jump zone!!!
4695	*/
4696	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4697	{
4698	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
4699	{
4700	PVM pVM = pVCpu->CTX_SUFF(pVM);
4701	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4702
4703	/*
4704	* Figure out fixed CR0 bits in VMX operation.
4705	*/
4706	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4707	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4708	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4709	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
4710	else
4711	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
4712
4713	if (!pVmxTransient->fIsNestedGuest)
4714	{
4715	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4716	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4717	uint64_t const u64ShadowCr0 = u64GuestCr0;
4718	Assert(!RT_HI_U32(u64GuestCr0));
4719
4720	/*
4721	* Setup VT-x's view of the guest CR0.
4722	*/
4723	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
4724	if (pVM->hm.s.fNestedPaging)
4725	{
4726	if (CPUMIsGuestPagingEnabled(pVCpu))
4727	{
4728	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
4729	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
4730	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
4731	}
4732	else
4733	{
4734	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
4735	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
4736	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
4737	}
4738
4739	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
4740	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4741	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
4742	}
4743	else
4744	{
4745	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
4746	u64GuestCr0 \|= X86_CR0_WP;
4747	}
4748
4749	/*
4750	* Guest FPU bits.
4751	*
4752	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
4753	* using CR0.TS.
4754	*
4755	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
4756	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
4757	*/
4758	u64GuestCr0 \|= X86_CR0_NE;
4759
4760	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
4761	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
4762
4763	/*
4764	* Update exception intercepts.
4765	*/
4766	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
4767	if (pVmcsInfo->RealMode.fRealOnV86Active)
4768	{
4769	Assert(PDMVmmDevHeapIsEnabled(pVM));
4770	Assert(pVM->hm.s.vmx.pRealModeTSS);
4771	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
4772	}
4773	else
4774	{
4775	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
4776	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
4777	if (fInterceptMF)
4778	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
4779	}
4780
4781	/* Additional intercepts for debugging, define these yourself explicitly. */
4782	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4783	uXcptBitmap \|= 0
4784	\| RT_BIT(X86_XCPT_BP)
4785	\| RT_BIT(X86_XCPT_DE)
4786	\| RT_BIT(X86_XCPT_NM)
4787	\| RT_BIT(X86_XCPT_TS)
4788	\| RT_BIT(X86_XCPT_UD)
4789	\| RT_BIT(X86_XCPT_NP)
4790	\| RT_BIT(X86_XCPT_SS)
4791	\| RT_BIT(X86_XCPT_GP)
4792	\| RT_BIT(X86_XCPT_PF)
4793	\| RT_BIT(X86_XCPT_MF)
4794	;
4795	#elif defined(HMVMX_ALWAYS_TRAP_PF)
4796	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
4797	#endif
4798	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
4799	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
4800	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
4801
4802	/* Apply the fixed CR0 bits and enable caching. */
4803	u64GuestCr0 \|= fSetCr0;
4804	u64GuestCr0 &= fZapCr0;
4805	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4806
4807	/* Commit the CR0 and related fields to the guest VMCS. */
4808	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo Fix to 64-bit when we drop 32-bit. */
4809	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4810	if (uProcCtls != pVmcsInfo->u32ProcCtls)
4811	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4812	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
4813	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
4814	AssertRCReturn(rc, rc);
4815
4816	/* Update our caches. */
4817	pVmcsInfo->u32ProcCtls = uProcCtls;
4818	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
4819
4820	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4821	}
4822	else
4823	{
4824	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
4825	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4826	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4827	uint64_t const u64ShadowCr0 = pVmcsNstGst->u64Cr0ReadShadow.u;
4828	Assert(!RT_HI_U32(u64GuestCr0));
4829	Assert(u64GuestCr0 & X86_CR0_NE);
4830
4831	/* Apply the fixed CR0 bits and enable caching. */
4832	u64GuestCr0 \|= fSetCr0;
4833	u64GuestCr0 &= fZapCr0;
4834	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4835
4836	/* Commit the CR0 and CR0 read shadow to the nested-guest VMCS. */
4837	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo NSTVMX: Fix to 64-bit when we drop 32-bit. */
4838	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4839	AssertRCReturn(rc, rc);
4840
4841	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4842	}
4843
4844	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
4845	}
4846
4847	return VINF_SUCCESS;
4848	}
4849
4850
4851	/**
4852	* Exports the guest control registers (CR3, CR4) into the guest-state area
4853	* in the VMCS.
4854	*
4855	* @returns VBox strict status code.
4856	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
4857	* without unrestricted guest access and the VMMDev is not presently
4858	* mapped (e.g. EFI32).
4859	*
4860	* @param pVCpu The cross context virtual CPU structure.
4861	* @param pVmxTransient The VMX-transient structure.
4862	*
4863	* @remarks No-long-jump zone!!!
4864	*/
4865	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4866	{
4867	int rc = VINF_SUCCESS;
4868	PVM pVM = pVCpu->CTX_SUFF(pVM);
4869
4870	/*
4871	* Guest CR2.
4872	* It's always loaded in the assembler code. Nothing to do here.
4873	*/
4874
4875	/*
4876	* Guest CR3.
4877	*/
4878	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
4879	{
4880	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
4881
4882	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
4883	if (pVM->hm.s.fNestedPaging)
4884	{
4885	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4886	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
4887
4888	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
4889	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
4890	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
4891	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
4892
4893	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
4894	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
4895	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
4896
4897	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
4898	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
4899	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
4900	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
4901	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
4902	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
4903	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
4904
4905	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
4906	AssertRCReturn(rc, rc);
4907
4908	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4909	if ( pVM->hm.s.vmx.fUnrestrictedGuest
4910	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
4911	{
4912	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
4913	if (CPUMIsGuestInPAEModeEx(pCtx))
4914	{
4915	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
4916	AssertRCReturn(rc, rc);
4917	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
4918	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
4919	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
4920	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
4921	AssertRCReturn(rc, rc);
4922	}
4923
4924	/*
4925	* The guest's view of its CR3 is unblemished with nested paging when the
4926	* guest is using paging or we have unrestricted guest execution to handle
4927	* the guest when it's not using paging.
4928	*/
4929	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
4930	GCPhysGuestCR3 = pCtx->cr3;
4931	}
4932	else
4933	{
4934	/*
4935	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
4936	* thinks it accesses physical memory directly, we use our identity-mapped
4937	* page table to map guest-linear to guest-physical addresses. EPT takes care
4938	* of translating it to host-physical addresses.
4939	*/
4940	RTGCPHYS GCPhys;
4941	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4942
4943	/* We obtain it here every time as the guest could have relocated this PCI region. */
4944	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4945	if (RT_SUCCESS(rc))
4946	{ /* likely */ }
4947	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4948	{
4949	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
4950	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4951	}
4952	else
4953	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4954
4955	GCPhysGuestCR3 = GCPhys;
4956	}
4957
4958	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
4959	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4960	AssertRCReturn(rc, rc);
4961	}
4962	else
4963	{
4964	/* Non-nested paging case, just use the hypervisor's CR3. */
4965	RTHCPHYS const HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4966
4967	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
4968	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4969	AssertRCReturn(rc, rc);
4970	}
4971
4972	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
4973	}
4974
4975	/*
4976	* Guest CR4.
4977	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4978	*/
4979	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
4980	{
4981	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4982	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4983	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
4984
4985	/*
4986	* Figure out fixed CR4 bits in VMX operation.
4987	*/
4988	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
4989	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
4990
4991	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
4992	uint64_t u64GuestCr4 = pCtx->cr4;
4993	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest ? pCtx->cr4 : pVmcsNstGst->u64Cr4ReadShadow.u;
4994	Assert(!RT_HI_U32(u64GuestCr4));
4995
4996	/*
4997	* Setup VT-x's view of the guest CR4.
4998	*
4999	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5000	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5001	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5002	*
5003	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5004	*/
5005	if (pVmcsInfo->RealMode.fRealOnV86Active)
5006	{
5007	Assert(pVM->hm.s.vmx.pRealModeTSS);
5008	Assert(PDMVmmDevHeapIsEnabled(pVM));
5009	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5010	}
5011
5012	if (pVM->hm.s.fNestedPaging)
5013	{
5014	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5015	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5016	{
5017	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5018	u64GuestCr4 \|= X86_CR4_PSE;
5019	/* Our identity mapping is a 32-bit page directory. */
5020	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5021	}
5022	/* else use guest CR4.*/
5023	}
5024	else
5025	{
5026	Assert(!pVmxTransient->fIsNestedGuest);
5027
5028	/*
5029	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5030	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5031	*/
5032	switch (pVCpu->hm.s.enmShadowMode)
5033	{
5034	case PGMMODE_REAL: /* Real-mode. */
5035	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5036	case PGMMODE_32_BIT: /* 32-bit paging. */
5037	{
5038	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5039	break;
5040	}
5041
5042	case PGMMODE_PAE: /* PAE paging. */
5043	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5044	{
5045	u64GuestCr4 \|= X86_CR4_PAE;
5046	break;
5047	}
5048
5049	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5050	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5051	#ifdef VBOX_ENABLE_64_BITS_GUESTS
5052	break;
5053	#endif
5054	default:
5055	AssertFailed();
5056	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5057	}
5058	}
5059
5060	/* Apply the fixed CR4 bits (mainly CR4.VMXE). */
5061	u64GuestCr4 \|= fSetCr4;
5062	u64GuestCr4 &= fZapCr4;
5063
5064	/* Commit the CR4 and CR4 read shadow to the guest VMCS. */
5065	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u64GuestCr4); /** @todo Fix to 64-bit when we drop 32-bit. */
5066	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
5067	AssertRCReturn(rc, rc);
5068
5069	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5070	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5071
5072	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5073
5074	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5075	}
5076	return rc;
5077	}
5078
5079
5080	/**
5081	* Exports the guest debug registers into the guest-state area in the VMCS.
5082	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5083	*
5084	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5085	*
5086	* @returns VBox status code.
5087	* @param pVCpu The cross context virtual CPU structure.
5088	* @param pVmxTransient The VMX-transient structure.
5089	*
5090	* @remarks No-long-jump zone!!!
5091	*/
5092	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5093	{
5094	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5095
5096	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5097	* stepping. */
5098	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5099	if (pVmxTransient->fIsNestedGuest)
5100	{
5101	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5102	AssertRCReturn(rc, rc);
5103	return VINF_SUCCESS;
5104	}
5105
5106	#ifdef VBOX_STRICT
5107	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5108	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5109	{
5110	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5111	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5112	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5113	}
5114	#endif
5115
5116	bool fSteppingDB = false;
5117	bool fInterceptMovDRx = false;
5118	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5119	if (pVCpu->hm.s.fSingleInstruction)
5120	{
5121	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5122	PVM pVM = pVCpu->CTX_SUFF(pVM);
5123	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5124	{
5125	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5126	Assert(fSteppingDB == false);
5127	}
5128	else
5129	{
5130	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5131	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5132	pVCpu->hm.s.fClearTrapFlag = true;
5133	fSteppingDB = true;
5134	}
5135	}
5136
5137	uint32_t u32GuestDr7;
5138	if ( fSteppingDB
5139	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5140	{
5141	/*
5142	* Use the combined guest and host DRx values found in the hypervisor register set
5143	* because the hypervisor debugger has breakpoints active or someone is single stepping
5144	* on the host side without a monitor trap flag.
5145	*
5146	* Note! DBGF expects a clean DR6 state before executing guest code.
5147	*/
5148	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5149	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5150	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
5151	{
5152	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5153	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
5154	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
5155	}
5156	else
5157	#endif
5158	if (!CPUMIsHyperDebugStateActive(pVCpu))
5159	{
5160	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5161	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5162	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5163	}
5164
5165	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5166	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
5167	pVCpu->hm.s.fUsingHyperDR7 = true;
5168	fInterceptMovDRx = true;
5169	}
5170	else
5171	{
5172	/*
5173	* If the guest has enabled debug registers, we need to load them prior to
5174	* executing guest code so they'll trigger at the right time.
5175	*/
5176	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5177	{
5178	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5179	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5180	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
5181	{
5182	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5183	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
5184	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
5185	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5186	}
5187	else
5188	#endif
5189	if (!CPUMIsGuestDebugStateActive(pVCpu))
5190	{
5191	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5192	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5193	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5194	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5195	}
5196	Assert(!fInterceptMovDRx);
5197	}
5198	/*
5199	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5200	* must intercept #DB in order to maintain a correct DR6 guest value, and
5201	* because we need to intercept it to prevent nested #DBs from hanging the
5202	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5203	*/
5204	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5205	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
5206	&& !CPUMIsGuestDebugStateActive(pVCpu))
5207	#else
5208	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5209	#endif
5210	{
5211	fInterceptMovDRx = true;
5212	}
5213
5214	/* Update DR7 with the actual guest value. */
5215	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5216	pVCpu->hm.s.fUsingHyperDR7 = false;
5217	}
5218
5219	if (fInterceptMovDRx)
5220	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5221	else
5222	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5223
5224	/*
5225	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5226	* monitor-trap flag and update our cache.
5227	*/
5228	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5229	{
5230	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5231	AssertRCReturn(rc2, rc2);
5232	pVmcsInfo->u32ProcCtls = uProcCtls;
5233	}
5234
5235	/*
5236	* Update guest DR7.
5237	*/
5238	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
5239	AssertRCReturn(rc, rc);
5240
5241	/*
5242	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
5243	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
5244	*
5245	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5246	*/
5247	if (fSteppingDB)
5248	{
5249	Assert(pVCpu->hm.s.fSingleInstruction);
5250	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
5251
5252	uint32_t fIntrState = 0;
5253	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
5254	AssertRCReturn(rc, rc);
5255
5256	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5257	{
5258	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5259	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5260	AssertRCReturn(rc, rc);
5261	}
5262	}
5263
5264	return VINF_SUCCESS;
5265	}
5266
5267
5268	#ifdef VBOX_STRICT
5269	/**
5270	* Strict function to validate segment registers.
5271	*
5272	* @param pVCpu The cross context virtual CPU structure.
5273	* @param pVmcsInfo The VMCS info. object.
5274	*
5275	* @remarks Will import guest CR0 on strict builds during validation of
5276	* segments.
5277	*/
5278	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
5279	{
5280	/*
5281	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5282	*
5283	* The reason we check for attribute value 0 in this function and not just the unusable bit is
5284	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
5285	* unusable bit and doesn't change the guest-context value.
5286	*/
5287	PVM pVM = pVCpu->CTX_SUFF(pVM);
5288	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5289	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
5290	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
5291	&& ( !CPUMIsGuestInRealModeEx(pCtx)
5292	&& !CPUMIsGuestInV86ModeEx(pCtx)))
5293	{
5294	/* Protected mode checks */
5295	/* CS */
5296	Assert(pCtx->cs.Attr.n.u1Present);
5297	Assert(!(pCtx->cs.Attr.u & 0xf00));
5298	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
5299	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5300	\|\| !(pCtx->cs.Attr.n.u1Granularity));
5301	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
5302	\|\| (pCtx->cs.Attr.n.u1Granularity));
5303	/* CS cannot be loaded with NULL in protected mode. */
5304	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
5305	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5306	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
5307	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5308	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
5309	else
5310	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
5311	/* SS */
5312	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5313	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
5314	if ( !(pCtx->cr0 & X86_CR0_PE)
5315	\|\| pCtx->cs.Attr.n.u4Type == 3)
5316	{
5317	Assert(!pCtx->ss.Attr.n.u2Dpl);
5318	}
5319	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
5320	{
5321	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5322	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
5323	Assert(pCtx->ss.Attr.n.u1Present);
5324	Assert(!(pCtx->ss.Attr.u & 0xf00));
5325	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
5326	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
5327	\|\| !(pCtx->ss.Attr.n.u1Granularity));
5328	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
5329	\|\| (pCtx->ss.Attr.n.u1Granularity));
5330	}
5331	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
5332	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
5333	{
5334	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5335	Assert(pCtx->ds.Attr.n.u1Present);
5336	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
5337	Assert(!(pCtx->ds.Attr.u & 0xf00));
5338	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
5339	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
5340	\|\| !(pCtx->ds.Attr.n.u1Granularity));
5341	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
5342	\|\| (pCtx->ds.Attr.n.u1Granularity));
5343	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5344	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
5345	}
5346	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
5347	{
5348	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5349	Assert(pCtx->es.Attr.n.u1Present);
5350	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
5351	Assert(!(pCtx->es.Attr.u & 0xf00));
5352	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
5353	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
5354	\|\| !(pCtx->es.Attr.n.u1Granularity));
5355	Assert( !(pCtx->es.u32Limit & 0xfff00000)
5356	\|\| (pCtx->es.Attr.n.u1Granularity));
5357	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5358	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
5359	}
5360	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
5361	{
5362	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5363	Assert(pCtx->fs.Attr.n.u1Present);
5364	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
5365	Assert(!(pCtx->fs.Attr.u & 0xf00));
5366	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
5367	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
5368	\|\| !(pCtx->fs.Attr.n.u1Granularity));
5369	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
5370	\|\| (pCtx->fs.Attr.n.u1Granularity));
5371	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5372	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5373	}
5374	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
5375	{
5376	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5377	Assert(pCtx->gs.Attr.n.u1Present);
5378	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
5379	Assert(!(pCtx->gs.Attr.u & 0xf00));
5380	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
5381	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
5382	\|\| !(pCtx->gs.Attr.n.u1Granularity));
5383	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
5384	\|\| (pCtx->gs.Attr.n.u1Granularity));
5385	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5386	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5387	}
5388	/* 64-bit capable CPUs. */
5389	# if HC_ARCH_BITS == 64
5390	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5391	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
5392	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
5393	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
5394	# endif
5395	}
5396	else if ( CPUMIsGuestInV86ModeEx(pCtx)
5397	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
5398	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
5399	{
5400	/* Real and v86 mode checks. */
5401	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
5402	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
5403	if (pVmcsInfo->RealMode.fRealOnV86Active)
5404	{
5405	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
5406	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
5407	}
5408	else
5409	{
5410	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
5411	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
5412	}
5413
5414	/* CS */
5415	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
5416	Assert(pCtx->cs.u32Limit == 0xffff);
5417	Assert(u32CSAttr == 0xf3);
5418	/* SS */
5419	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
5420	Assert(pCtx->ss.u32Limit == 0xffff);
5421	Assert(u32SSAttr == 0xf3);
5422	/* DS */
5423	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
5424	Assert(pCtx->ds.u32Limit == 0xffff);
5425	Assert(u32DSAttr == 0xf3);
5426	/* ES */
5427	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
5428	Assert(pCtx->es.u32Limit == 0xffff);
5429	Assert(u32ESAttr == 0xf3);
5430	/* FS */
5431	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
5432	Assert(pCtx->fs.u32Limit == 0xffff);
5433	Assert(u32FSAttr == 0xf3);
5434	/* GS */
5435	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
5436	Assert(pCtx->gs.u32Limit == 0xffff);
5437	Assert(u32GSAttr == 0xf3);
5438	/* 64-bit capable CPUs. */
5439	# if HC_ARCH_BITS == 64
5440	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5441	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
5442	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
5443	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
5444	# endif
5445	}
5446	}
5447	#endif /* VBOX_STRICT */
5448
5449
5450	/**
5451	* Exports a guest segment register into the guest-state area in the VMCS.
5452	*
5453	* @returns VBox status code.
5454	* @param pVCpu The cross context virtual CPU structure.
5455	* @param pVmcsInfo The VMCS info. object.
5456	* @param iSegReg The segment register number (X86_SREG_XXX).
5457	* @param pSelReg Pointer to the segment selector.
5458	*
5459	* @remarks No-long-jump zone!!!
5460	*/
5461	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
5462	{
5463	Assert(iSegReg < X86_SREG_COUNT);
5464	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
5465	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
5466	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
5467	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
5468
5469	uint32_t u32Access = pSelReg->Attr.u;
5470	if (pVmcsInfo->RealMode.fRealOnV86Active)
5471	{
5472	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
5473	u32Access = 0xf3;
5474	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5475	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5476	RT_NOREF_PV(pVCpu);
5477	}
5478	else
5479	{
5480	/*
5481	* The way to differentiate between whether this is really a null selector or was just
5482	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
5483	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
5484	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
5485	* NULL selectors loaded in protected-mode have their attribute as 0.
5486	*/
5487	if (!u32Access)
5488	u32Access = X86DESCATTR_UNUSABLE;
5489	}
5490
5491	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
5492	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
5493	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
5494
5495	/*
5496	* Commit it to the VMCS.
5497	*/
5498	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
5499	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
5500	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
5501	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
5502	AssertRCReturn(rc, rc);
5503	return rc;
5504	}
5505
5506
5507	/**
5508	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
5509	* area in the VMCS.
5510	*
5511	* @returns VBox status code.
5512	* @param pVCpu The cross context virtual CPU structure.
5513	* @param pVmxTransient The VMX-transient structure.
5514	*
5515	* @remarks Will import guest CR0 on strict builds during validation of
5516	* segments.
5517	* @remarks No-long-jump zone!!!
5518	*/
5519	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5520	{
5521	int rc = VERR_INTERNAL_ERROR_5;
5522	PVM pVM = pVCpu->CTX_SUFF(pVM);
5523	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5524	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5525
5526	/*
5527	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
5528	*/
5529	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
5530	{
5531	#ifdef VBOX_WITH_REM
5532	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
5533	{
5534	Assert(!pVmxTransient->fIsNestedGuest);
5535	Assert(pVM->hm.s.vmx.pRealModeTSS);
5536	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
5537	if ( pVmcsInfo->fWasInRealMode
5538	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
5539	{
5540	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
5541	in real-mode (e.g. OpenBSD 4.0) */
5542	REMFlushTBs(pVM);
5543	Log4Func(("Switch to protected mode detected!\n"));
5544	pVmcsInfo->fWasInRealMode = false;
5545	}
5546	}
5547	#endif
5548	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
5549	{
5550	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
5551	if (pVmcsInfo->RealMode.fRealOnV86Active)
5552	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
5553	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
5554	AssertRCReturn(rc, rc);
5555	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
5556	}
5557
5558	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
5559	{
5560	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
5561	if (pVmcsInfo->RealMode.fRealOnV86Active)
5562	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
5563	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
5564	AssertRCReturn(rc, rc);
5565	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
5566	}
5567
5568	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
5569	{
5570	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
5571	if (pVmcsInfo->RealMode.fRealOnV86Active)
5572	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
5573	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
5574	AssertRCReturn(rc, rc);
5575	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
5576	}
5577
5578	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
5579	{
5580	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
5581	if (pVmcsInfo->RealMode.fRealOnV86Active)
5582	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
5583	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
5584	AssertRCReturn(rc, rc);
5585	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
5586	}
5587
5588	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
5589	{
5590	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
5591	if (pVmcsInfo->RealMode.fRealOnV86Active)
5592	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
5593	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
5594	AssertRCReturn(rc, rc);
5595	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
5596	}
5597
5598	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
5599	{
5600	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
5601	if (pVmcsInfo->RealMode.fRealOnV86Active)
5602	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
5603	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
5604	AssertRCReturn(rc, rc);
5605	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
5606	}
5607
5608	#ifdef VBOX_STRICT
5609	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
5610	#endif
5611	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
5612	pCtx->cs.Attr.u));
5613	}
5614
5615	/*
5616	* Guest TR.
5617	*/
5618	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
5619	{
5620	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
5621
5622	/*
5623	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
5624	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
5625	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
5626	*/
5627	uint16_t u16Sel;
5628	uint32_t u32Limit;
5629	uint64_t u64Base;
5630	uint32_t u32AccessRights;
5631	if (!pVmcsInfo->RealMode.fRealOnV86Active)
5632	{
5633	u16Sel = pCtx->tr.Sel;
5634	u32Limit = pCtx->tr.u32Limit;
5635	u64Base = pCtx->tr.u64Base;
5636	u32AccessRights = pCtx->tr.Attr.u;
5637	}
5638	else
5639	{
5640	Assert(!pVmxTransient->fIsNestedGuest);
5641	Assert(pVM->hm.s.vmx.pRealModeTSS);
5642	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
5643
5644	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
5645	RTGCPHYS GCPhys;
5646	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
5647	AssertRCReturn(rc, rc);
5648
5649	X86DESCATTR DescAttr;
5650	DescAttr.u = 0;
5651	DescAttr.n.u1Present = 1;
5652	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
5653
5654	u16Sel = 0;
5655	u32Limit = HM_VTX_TSS_SIZE;
5656	u64Base = GCPhys;
5657	u32AccessRights = DescAttr.u;
5658	}
5659
5660	/* Validate. */
5661	Assert(!(u16Sel & RT_BIT(2)));
5662	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
5663	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
5664	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
5665	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
5666	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
5667	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
5668	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
5669	Assert( (u32Limit & 0xfff) == 0xfff
5670	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
5671	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
5672	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
5673
5674	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
5675	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
5676	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
5677	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
5678	AssertRCReturn(rc, rc);
5679
5680	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
5681	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
5682	}
5683
5684	/*
5685	* Guest GDTR.
5686	*/
5687	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
5688	{
5689	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
5690
5691	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
5692	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
5693	AssertRCReturn(rc, rc);
5694
5695	/* Validate. */
5696	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5697
5698	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
5699	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
5700	}
5701
5702	/*
5703	* Guest LDTR.
5704	*/
5705	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
5706	{
5707	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
5708
5709	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
5710	uint32_t u32Access;
5711	if ( !pVmxTransient->fIsNestedGuest
5712	&& !pCtx->ldtr.Attr.u)
5713	u32Access = X86DESCATTR_UNUSABLE;
5714	else
5715	u32Access = pCtx->ldtr.Attr.u;
5716
5717	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
5718	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
5719	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
5720	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
5721	AssertRCReturn(rc, rc);
5722
5723	/* Validate. */
5724	if (!(u32Access & X86DESCATTR_UNUSABLE))
5725	{
5726	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
5727	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
5728	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
5729	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
5730	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
5731	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
5732	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
5733	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
5734	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
5735	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
5736	}
5737
5738	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
5739	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
5740	}
5741
5742	/*
5743	* Guest IDTR.
5744	*/
5745	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
5746	{
5747	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
5748
5749	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
5750	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
5751	AssertRCReturn(rc, rc);
5752
5753	/* Validate. */
5754	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5755
5756	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
5757	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
5758	}
5759
5760	return VINF_SUCCESS;
5761	}
5762
5763
5764	/**
5765	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
5766	* areas.
5767	*
5768	* These MSRs will automatically be loaded to the host CPU on every successful
5769	* VM-entry and stored from the host CPU on every successful VM-exit.
5770	*
5771	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
5772	* actual host MSR values are not- updated here for performance reasons. See
5773	* hmR0VmxExportHostMsrs().
5774	*
5775	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
5776	*
5777	* @returns VBox status code.
5778	* @param pVCpu The cross context virtual CPU structure.
5779	* @param pVmxTransient The VMX-transient structure.
5780	*
5781	* @remarks No-long-jump zone!!!
5782	*/
5783	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5784	{
5785	AssertPtr(pVCpu);
5786	AssertPtr(pVmxTransient);
5787
5788	PVM pVM = pVCpu->CTX_SUFF(pVM);
5789	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5790
5791	/*
5792	* MSRs that we use the auto-load/store MSR area in the VMCS.
5793	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
5794	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
5795	*
5796	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
5797	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
5798	* emulation, nothing to do here.
5799	*/
5800	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
5801	{
5802	if ( !pVmxTransient->fIsNestedGuest
5803	&& pVM->hm.s.fAllow64BitGuests)
5804	{
5805	#if HC_ARCH_BITS == 32
5806	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
5807	Assert(!pVmxTransient->fIsNestedGuest);
5808
5809	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
5810	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
5811	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
5812	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
5813	AssertRCReturn(rc, rc);
5814	#endif
5815	}
5816	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
5817	}
5818
5819	/*
5820	* Guest Sysenter MSRs.
5821	*/
5822	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
5823	{
5824	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
5825
5826	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
5827	{
5828	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
5829	AssertRCReturn(rc, rc);
5830	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
5831	}
5832
5833	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
5834	{
5835	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
5836	AssertRCReturn(rc, rc);
5837	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
5838	}
5839
5840	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
5841	{
5842	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
5843	AssertRCReturn(rc, rc);
5844	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
5845	}
5846	}
5847
5848	/*
5849	* Guest/host EFER MSR.
5850	*/
5851	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
5852	{
5853	/* Whether we are using the VMCS to swap the EFER MSR must have been
5854	determined earlier while exporting VM-entry/VM-exit controls. */
5855	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
5856	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
5857
5858	if (hmR0VmxShouldSwapEferMsr(pVCpu))
5859	{
5860	/*
5861	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
5862	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
5863	*/
5864	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
5865	{
5866	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
5867	AssertRCReturn(rc, rc);
5868	}
5869	else
5870	{
5871	/*
5872	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
5873	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
5874	*/
5875	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
5876	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5877	AssertRCReturn(rc, rc);
5878	}
5879	}
5880	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
5881	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
5882
5883	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
5884	}
5885
5886	/*
5887	* Other MSRs.
5888	* Speculation Control (R/W).
5889	*/
5890	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
5891	{
5892	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
5893	if (pVM->cpum.ro.GuestFeatures.fIbrs)
5894	{
5895	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
5896	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5897	AssertRCReturn(rc, rc);
5898	}
5899	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
5900	}
5901
5902	return VINF_SUCCESS;
5903	}
5904
5905
5906	/**
5907	* Selects up the appropriate function to run guest code.
5908	*
5909	* @returns VBox status code.
5910	* @param pVCpu The cross context virtual CPU structure.
5911	* @param pVmxTransient The VMX-transient structure.
5912	*
5913	* @remarks No-long-jump zone!!!
5914	*/
5915	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5916	{
5917	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5918	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5919
5920	if (CPUMIsGuestInLongModeEx(pCtx))
5921	{
5922	#ifndef VBOX_ENABLE_64_BITS_GUESTS
5923	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5924	#endif
5925	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
5926	#if HC_ARCH_BITS == 32
5927	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
5928	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
5929	{
5930	#ifdef VBOX_STRICT
5931	if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5932	{
5933	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5934	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5935	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5936	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5937	("fCtxChanged=%#RX64\n", fCtxChanged));
5938	}
5939	#endif
5940	pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
5941
5942	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
5943	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
5944	pVmcsInfo->fSwitchedTo64on32 = true;
5945	Log4Func(("Selected 64-bit switcher\n"));
5946	}
5947	#else
5948	/* 64-bit host. */
5949	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
5950	#endif
5951	}
5952	else
5953	{
5954	/* Guest is not in long mode, use the 32-bit handler. */
5955	#if HC_ARCH_BITS == 32
5956	if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
5957	&& !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
5958	&& pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5959	{
5960	# ifdef VBOX_STRICT
5961	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5962	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5963	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5964	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5965	("fCtxChanged=%#RX64\n", fCtxChanged));
5966	# endif
5967	}
5968	# ifdef VBOX_ENABLE_64_BITS_GUESTS
5969	/*
5970	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
5971	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
5972	* switcher flag now because we know the guest is in a sane state where it's safe
5973	* to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
5974	* the much faster 32-bit switcher again.
5975	*/
5976	if (!pVmcsInfo->fSwitchedTo64on32)
5977	{
5978	if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
5979	Log4Func(("Selected 32-bit switcher\n"));
5980	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5981	}
5982	else
5983	{
5984	Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
5985	if ( pVmcsInfo->RealMode.fRealOnV86Active
5986	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
5987	{
5988	pVmcsInfo->fSwitchedTo64on32 = false;
5989	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5990	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
5991	\| HM_CHANGED_VMX_ENTRY_EXIT_CTLS
5992	\| HM_CHANGED_HOST_CONTEXT);
5993	Log4Func(("Selected 32-bit switcher (safe)\n"));
5994	}
5995	}
5996	# else
5997	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5998	# endif
5999	#else
6000	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6001	#endif
6002	}
6003	Assert(pVmcsInfo->pfnStartVM);
6004	return VINF_SUCCESS;
6005	}
6006
6007
6008	/**
6009	* Wrapper for running the guest code in VT-x.
6010	*
6011	* @returns VBox status code, no informational status codes.
6012	* @param pVCpu The cross context virtual CPU structure.
6013	* @param pVmxTransient The VMX-transient structure.
6014	*
6015	* @remarks No-long-jump zone!!!
6016	*/
6017	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6018	{
6019	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6020	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6021	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6022
6023	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6024
6025	/*
6026	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6027	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6028	* callee-saved and thus the need for this XMM wrapper.
6029	*
6030	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6031	*/
6032	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6033	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6034	PVM pVM = pVCpu->CTX_SUFF(pVM);
6035	#ifdef VBOX_WITH_KERNEL_USING_XMM
6036	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6037	#else
6038	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
6039	#endif
6040	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6041	return rc;
6042	}
6043
6044
6045	/**
6046	* Reports world-switch error and dumps some useful debug info.
6047	*
6048	* @param pVCpu The cross context virtual CPU structure.
6049	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6050	* @param pVmxTransient The VMX-transient structure (only
6051	* exitReason updated).
6052	*/
6053	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6054	{
6055	Assert(pVCpu);
6056	Assert(pVmxTransient);
6057	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6058
6059	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6060	switch (rcVMRun)
6061	{
6062	case VERR_VMX_INVALID_VMXON_PTR:
6063	AssertFailed();
6064	break;
6065	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6066	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6067	{
6068	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6069	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6070	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
6071	AssertRC(rc);
6072
6073	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6074	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6075	Cannot do it here as we may have been long preempted. */
6076
6077	#ifdef VBOX_STRICT
6078	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6079	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6080	pVmxTransient->uExitReason));
6081	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6082	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6083	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6084	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6085	else
6086	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6087	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6088	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6089
6090	/* VMX control bits. */
6091	uint32_t u32Val;
6092	uint64_t u64Val;
6093	RTHCUINTREG uHCReg;
6094	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
6095	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
6096	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
6097	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
6098	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
6099	{
6100	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
6101	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
6102	}
6103	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
6104	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
6105	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
6106	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
6107	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
6108	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
6109	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
6110	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
6111	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
6112	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
6113	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
6114	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
6115	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
6116	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
6117	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
6118	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
6119	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6120	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
6121	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6122	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
6123	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
6124	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
6125	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
6126	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
6127	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
6128	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
6129	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
6130	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
6131	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
6132	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6133	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
6134	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
6135	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
6136	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6137	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6138	{
6139	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
6140	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
6141	}
6142
6143	/* Guest bits. */
6144	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
6145	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
6146	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
6147	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
6148	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
6149	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
6150	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
6151	{
6152	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
6153	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
6154	}
6155
6156	/* Host bits. */
6157	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
6158	Log4(("Host CR0 %#RHr\n", uHCReg));
6159	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
6160	Log4(("Host CR3 %#RHr\n", uHCReg));
6161	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
6162	Log4(("Host CR4 %#RHr\n", uHCReg));
6163
6164	RTGDTR HostGdtr;
6165	PCX86DESCHC pDesc;
6166	ASMGetGDTR(&HostGdtr);
6167	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
6168	Log4(("Host CS %#08x\n", u32Val));
6169	if (u32Val < HostGdtr.cbGdt)
6170	{
6171	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6172	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
6173	}
6174
6175	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
6176	Log4(("Host DS %#08x\n", u32Val));
6177	if (u32Val < HostGdtr.cbGdt)
6178	{
6179	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6180	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
6181	}
6182
6183	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
6184	Log4(("Host ES %#08x\n", u32Val));
6185	if (u32Val < HostGdtr.cbGdt)
6186	{
6187	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6188	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
6189	}
6190
6191	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
6192	Log4(("Host FS %#08x\n", u32Val));
6193	if (u32Val < HostGdtr.cbGdt)
6194	{
6195	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6196	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
6197	}
6198
6199	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
6200	Log4(("Host GS %#08x\n", u32Val));
6201	if (u32Val < HostGdtr.cbGdt)
6202	{
6203	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6204	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
6205	}
6206
6207	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
6208	Log4(("Host SS %#08x\n", u32Val));
6209	if (u32Val < HostGdtr.cbGdt)
6210	{
6211	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6212	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
6213	}
6214
6215	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
6216	Log4(("Host TR %#08x\n", u32Val));
6217	if (u32Val < HostGdtr.cbGdt)
6218	{
6219	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6220	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
6221	}
6222
6223	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
6224	Log4(("Host TR Base %#RHv\n", uHCReg));
6225	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
6226	Log4(("Host GDTR Base %#RHv\n", uHCReg));
6227	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
6228	Log4(("Host IDTR Base %#RHv\n", uHCReg));
6229	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
6230	Log4(("Host SYSENTER CS %#08x\n", u32Val));
6231	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
6232	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
6233	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
6234	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
6235	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
6236	Log4(("Host RSP %#RHv\n", uHCReg));
6237	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
6238	Log4(("Host RIP %#RHv\n", uHCReg));
6239	# if HC_ARCH_BITS == 64
6240	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6241	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6242	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6243	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6244	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6245	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6246	# endif
6247	#endif /* VBOX_STRICT */
6248	break;
6249	}
6250
6251	default:
6252	/* Impossible */
6253	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6254	break;
6255	}
6256	}
6257
6258
6259	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
6260	# ifndef VMX_USE_CACHED_VMCS_ACCESSES
6261	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
6262	# endif
6263
6264	/**
6265	* Initialize the VMCS-Read cache.
6266	*
6267	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
6268	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
6269	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
6270	* (those that have a 32-bit FULL & HIGH part).
6271	*
6272	* @param pVCpu The cross context virtual CPU structure.
6273	*/
6274	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
6275	{
6276	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
6277	do { \
6278	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
6279	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
6280	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
6281	++cReadFields; \
6282	} while (0)
6283
6284	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6285	uint32_t cReadFields = 0;
6286
6287	/*
6288	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
6289	* and serve to indicate exceptions to the rules.
6290	*/
6291
6292	/* Guest-natural selector base fields. */
6293	#if 0
6294	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
6295	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
6296	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
6297	#endif
6298	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
6299	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
6300	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
6301	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
6302	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
6303	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
6304	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
6305	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
6306	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
6307	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
6308	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
6309	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
6310	#if 0
6311	/* Unused natural width guest-state fields. */
6312	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
6313	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
6314	#endif
6315	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
6316	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
6317
6318	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
6319	these 64-bit fields (using "FULL" and "HIGH" fields). */
6320	#if 0
6321	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
6322	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
6323	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
6324	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
6325	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
6326	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
6327	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
6328	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
6329	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
6330	#endif
6331
6332	/* Natural width guest-state fields. */
6333	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
6334	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
6335
6336	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6337	{
6338	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
6339	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
6340	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
6341	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
6342	}
6343	else
6344	{
6345	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
6346	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
6347	}
6348
6349	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
6350	}
6351
6352
6353	/**
6354	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
6355	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
6356	* darwin, running 64-bit guests).
6357	*
6358	* @returns VBox status code.
6359	* @param pVCpu The cross context virtual CPU structure.
6360	* @param idxField The VMCS field encoding.
6361	* @param u64Val 16, 32 or 64-bit value.
6362	*/
6363	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6364	{
6365	int rc;
6366	switch (idxField)
6367	{
6368	/*
6369	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
6370	*/
6371	/* 64-bit Control fields. */
6372	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
6373	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
6374	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
6375	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
6376	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
6377	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
6378	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
6379	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
6380	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
6381	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
6382	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
6383	case VMX_VMCS64_CTRL_EPTP_FULL:
6384	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
6385	/* 64-bit Guest-state fields. */
6386	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
6387	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
6388	case VMX_VMCS64_GUEST_PAT_FULL:
6389	case VMX_VMCS64_GUEST_EFER_FULL:
6390	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
6391	case VMX_VMCS64_GUEST_PDPTE0_FULL:
6392	case VMX_VMCS64_GUEST_PDPTE1_FULL:
6393	case VMX_VMCS64_GUEST_PDPTE2_FULL:
6394	case VMX_VMCS64_GUEST_PDPTE3_FULL:
6395	/* 64-bit Host-state fields. */
6396	case VMX_VMCS64_HOST_PAT_FULL:
6397	case VMX_VMCS64_HOST_EFER_FULL:
6398	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
6399	{
6400	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6401	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
6402	break;
6403	}
6404
6405	/*
6406	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
6407	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
6408	*/
6409	/* Natural-width Guest-state fields. */
6410	case VMX_VMCS_GUEST_CR3:
6411	case VMX_VMCS_GUEST_ES_BASE:
6412	case VMX_VMCS_GUEST_CS_BASE:
6413	case VMX_VMCS_GUEST_SS_BASE:
6414	case VMX_VMCS_GUEST_DS_BASE:
6415	case VMX_VMCS_GUEST_FS_BASE:
6416	case VMX_VMCS_GUEST_GS_BASE:
6417	case VMX_VMCS_GUEST_LDTR_BASE:
6418	case VMX_VMCS_GUEST_TR_BASE:
6419	case VMX_VMCS_GUEST_GDTR_BASE:
6420	case VMX_VMCS_GUEST_IDTR_BASE:
6421	case VMX_VMCS_GUEST_RSP:
6422	case VMX_VMCS_GUEST_RIP:
6423	case VMX_VMCS_GUEST_SYSENTER_ESP:
6424	case VMX_VMCS_GUEST_SYSENTER_EIP:
6425	{
6426	if (!(RT_HI_U32(u64Val)))
6427	{
6428	/* If this field is 64-bit, VT-x will zero out the top bits. */
6429	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6430	}
6431	else
6432	{
6433	/* Assert that only the 32->64 switcher case should ever come here. */
6434	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
6435	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
6436	}
6437	break;
6438	}
6439
6440	default:
6441	{
6442	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
6443	pVCpu->hm.s.u32HMError = idxField;
6444	rc = VERR_INVALID_PARAMETER;
6445	break;
6446	}
6447	}
6448	AssertRCReturn(rc, rc);
6449	return rc;
6450	}
6451
6452
6453	/**
6454	* Queue up a VMWRITE by using the VMCS write cache.
6455	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
6456	*
6457	* @param pVCpu The cross context virtual CPU structure.
6458	* @param idxField The VMCS field encoding.
6459	* @param u64Val 16, 32 or 64-bit value.
6460	*/
6461	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6462	{
6463	AssertPtr(pVCpu);
6464	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6465
6466	AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
6467	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
6468
6469	/* Make sure there are no duplicates. */
6470	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
6471	{
6472	if (pCache->Write.aField[i] == idxField)
6473	{
6474	pCache->Write.aFieldVal[i] = u64Val;
6475	return VINF_SUCCESS;
6476	}
6477	}
6478
6479	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
6480	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
6481	pCache->Write.cValidEntries++;
6482	return VINF_SUCCESS;
6483	}
6484	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
6485
6486
6487	/**
6488	* Sets up the usage of TSC-offsetting and updates the VMCS.
6489	*
6490	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6491	* VMX-preemption timer.
6492	*
6493	* @returns VBox status code.
6494	* @param pVCpu The cross context virtual CPU structure.
6495	* @param pVmxTransient The VMX-transient structure.
6496	*
6497	* @remarks No-long-jump zone!!!
6498	*/
6499	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6500	{
6501	bool fOffsettedTsc;
6502	bool fParavirtTsc;
6503	uint64_t uTscOffset;
6504	PVM pVM = pVCpu->CTX_SUFF(pVM);
6505	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6506
6507	if (pVM->hm.s.vmx.fUsePreemptTimer)
6508	{
6509	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6510
6511	/* Make sure the returned values have sane upper and lower boundaries. */
6512	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6513	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6514	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6515	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6516
6517	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6518	* preemption timers here. We probably need to clamp the preemption timer,
6519	* after converting the timer value to the host. */
6520	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6521	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6522	AssertRC(rc);
6523	}
6524	else
6525	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6526
6527	if (fParavirtTsc)
6528	{
6529	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6530	information before every VM-entry, hence disable it for performance sake. */
6531	#if 0
6532	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
6533	AssertRC(rc);
6534	#endif
6535	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
6536	}
6537
6538	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
6539	if ( fOffsettedTsc
6540	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
6541	{
6542	if (pVmxTransient->fIsNestedGuest)
6543	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
6544	if (pVmcsInfo->u64TscOffset != uTscOffset)
6545	{
6546	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
6547	AssertRC(rc);
6548	pVmcsInfo->u64TscOffset = uTscOffset;
6549	}
6550
6551	if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
6552	{
6553	uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
6554	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6555	AssertRC(rc);
6556	pVmcsInfo->u32ProcCtls = uProcCtls;
6557	}
6558	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
6559	}
6560	else
6561	{
6562	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
6563	if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
6564	{
6565	uProcCtls \|= VMX_PROC_CTLS_RDTSC_EXIT;
6566	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6567	AssertRC(rc);
6568	pVmcsInfo->u32ProcCtls = uProcCtls;
6569	}
6570	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
6571	}
6572	}
6573
6574
6575	/**
6576	* Gets the IEM exception flags for the specified vector and IDT vectoring /
6577	* VM-exit interruption info type.
6578	*
6579	* @returns The IEM exception flags.
6580	* @param uVector The event vector.
6581	* @param uVmxEventType The VMX event type.
6582	*
6583	* @remarks This function currently only constructs flags required for
6584	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
6585	* and CR2 aspects of an exception are not included).
6586	*/
6587	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
6588	{
6589	uint32_t fIemXcptFlags;
6590	switch (uVmxEventType)
6591	{
6592	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6593	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6594	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6595	break;
6596
6597	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6598	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6599	break;
6600
6601	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6602	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
6603	break;
6604
6605	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
6606	{
6607	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6608	if (uVector == X86_XCPT_BP)
6609	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
6610	else if (uVector == X86_XCPT_OF)
6611	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
6612	else
6613	{
6614	fIemXcptFlags = 0;
6615	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
6616	}
6617	break;
6618	}
6619
6620	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6621	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6622	break;
6623
6624	default:
6625	fIemXcptFlags = 0;
6626	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
6627	break;
6628	}
6629	return fIemXcptFlags;
6630	}
6631
6632
6633	/**
6634	* Sets an event as a pending event to be injected into the guest.
6635	*
6636	* @param pVCpu The cross context virtual CPU structure.
6637	* @param u32IntInfo The VM-entry interruption-information field.
6638	* @param cbInstr The VM-entry instruction length in bytes (for software
6639	* interrupts, exceptions and privileged software
6640	* exceptions).
6641	* @param u32ErrCode The VM-entry exception error code.
6642	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
6643	* page-fault.
6644	*/
6645	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
6646	RTGCUINTPTR GCPtrFaultAddress)
6647	{
6648	Assert(!pVCpu->hm.s.Event.fPending);
6649	pVCpu->hm.s.Event.fPending = true;
6650	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
6651	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
6652	pVCpu->hm.s.Event.cbInstr = cbInstr;
6653	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
6654	}
6655
6656
6657	/**
6658	* Sets an external interrupt as pending-for-injection into the VM.
6659	*
6660	* @param pVCpu The cross context virtual CPU structure.
6661	* @param u8Interrupt The external interrupt vector.
6662	*/
6663	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
6664	{
6665	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
6666	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6667	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6668	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6669	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6670	}
6671
6672
6673	/**
6674	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
6675	*
6676	* @param pVCpu The cross context virtual CPU structure.
6677	*/
6678	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
6679	{
6680	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
6681	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
6682	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6683	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6684	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6685	}
6686
6687
6688	/**
6689	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
6690	*
6691	* @param pVCpu The cross context virtual CPU structure.
6692	*/
6693	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
6694	{
6695	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
6696	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6697	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6698	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6699	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6700	}
6701
6702
6703	/**
6704	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
6705	*
6706	* @param pVCpu The cross context virtual CPU structure.
6707	*/
6708	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
6709	{
6710	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
6711	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6712	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6713	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6714	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6715	}
6716
6717
6718	/**
6719	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
6720	*
6721	* @param pVCpu The cross context virtual CPU structure.
6722	*/
6723	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
6724	{
6725	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
6726	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6727	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6728	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6729	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6730	}
6731
6732
6733	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6734	/**
6735	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
6736	*
6737	* @param pVCpu The cross context virtual CPU structure.
6738	* @param u32ErrCode The error code for the general-protection exception.
6739	*/
6740	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
6741	{
6742	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
6743	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6744	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6745	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6746	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6747	}
6748
6749
6750	/**
6751	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
6752	*
6753	* @param pVCpu The cross context virtual CPU structure.
6754	* @param u32ErrCode The error code for the stack exception.
6755	*/
6756	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
6757	{
6758	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
6759	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6760	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6761	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6762	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6763	}
6764
6765
6766	/**
6767	* Decodes the memory operand of an instruction that caused a VM-exit.
6768	*
6769	* The VM-exit qualification field provides the displacement field for memory
6770	* operand instructions, if any.
6771	*
6772	* @returns Strict VBox status code (i.e. informational status codes too).
6773	* @retval VINF_SUCCESS if the operand was successfully decoded.
6774	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
6775	* operand.
6776	* @param pVCpu The cross context virtual CPU structure.
6777	* @param uExitInstrInfo The VM-exit instruction information field.
6778	* @param enmMemAccess The memory operand's access type (read or write).
6779	* @param GCPtrDisp The instruction displacement field, if any. For
6780	* RIP-relative addressing pass RIP + displacement here.
6781	* @param pGCPtrMem Where to store the effective destination memory address.
6782	*/
6783	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
6784	PRTGCPTR pGCPtrMem)
6785	{
6786	Assert(pGCPtrMem);
6787	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
6788	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
6789	\| CPUMCTX_EXTRN_CR0);
6790
6791	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
6792	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
6793	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
6794
6795	VMXEXITINSTRINFO ExitInstrInfo;
6796	ExitInstrInfo.u = uExitInstrInfo;
6797	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
6798	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
6799	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
6800	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
6801	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
6802	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
6803	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
6804	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
6805	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
6806
6807	/*
6808	* Validate instruction information.
6809	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
6810	*/
6811	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
6812	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
6813	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
6814	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
6815	AssertLogRelMsgReturn(fIsMemOperand,
6816	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
6817
6818	/*
6819	* Compute the complete effective address.
6820	*
6821	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
6822	* See AMD spec. 4.5.2 "Segment Registers".
6823	*/
6824	RTGCPTR GCPtrMem = GCPtrDisp;
6825	if (fBaseRegValid)
6826	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
6827	if (fIdxRegValid)
6828	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
6829
6830	RTGCPTR const GCPtrOff = GCPtrMem;
6831	if ( !fIsLongMode
6832	\|\| iSegReg >= X86_SREG_FS)
6833	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
6834	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
6835
6836	/*
6837	* Validate effective address.
6838	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
6839	*/
6840	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
6841	Assert(cbAccess > 0);
6842	if (fIsLongMode)
6843	{
6844	if (X86_IS_CANONICAL(GCPtrMem))
6845	{
6846	*pGCPtrMem = GCPtrMem;
6847	return VINF_SUCCESS;
6848	}
6849
6850	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
6851	* "Data Limit Checks in 64-bit Mode". */
6852	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
6853	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6854	return VINF_HM_PENDING_XCPT;
6855	}
6856
6857	/*
6858	* This is a watered down version of iemMemApplySegment().
6859	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
6860	* and segment CPL/DPL checks are skipped.
6861	*/
6862	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
6863	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
6864	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
6865
6866	/* Check if the segment is present and usable. */
6867	if ( pSel->Attr.n.u1Present
6868	&& !pSel->Attr.n.u1Unusable)
6869	{
6870	Assert(pSel->Attr.n.u1DescType);
6871	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
6872	{
6873	/* Check permissions for the data segment. */
6874	if ( enmMemAccess == VMXMEMACCESS_WRITE
6875	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
6876	{
6877	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6878	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
6879	return VINF_HM_PENDING_XCPT;
6880	}
6881
6882	/* Check limits if it's a normal data segment. */
6883	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
6884	{
6885	if ( GCPtrFirst32 > pSel->u32Limit
6886	\|\| GCPtrLast32 > pSel->u32Limit)
6887	{
6888	Log4Func(("Data segment limit exceeded."
6889	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6890	GCPtrLast32, pSel->u32Limit));
6891	if (iSegReg == X86_SREG_SS)
6892	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6893	else
6894	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6895	return VINF_HM_PENDING_XCPT;
6896	}
6897	}
6898	else
6899	{
6900	/* Check limits if it's an expand-down data segment.
6901	Note! The upper boundary is defined by the B bit, not the G bit! */
6902	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
6903	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
6904	{
6905	Log4Func(("Expand-down data segment limit exceeded."
6906	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6907	GCPtrLast32, pSel->u32Limit));
6908	if (iSegReg == X86_SREG_SS)
6909	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6910	else
6911	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6912	return VINF_HM_PENDING_XCPT;
6913	}
6914	}
6915	}
6916	else
6917	{
6918	/* Check permissions for the code segment. */
6919	if ( enmMemAccess == VMXMEMACCESS_WRITE
6920	\|\| ( enmMemAccess == VMXMEMACCESS_READ
6921	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
6922	{
6923	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6924	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6925	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6926	return VINF_HM_PENDING_XCPT;
6927	}
6928
6929	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6930	if ( GCPtrFirst32 > pSel->u32Limit
6931	\|\| GCPtrLast32 > pSel->u32Limit)
6932	{
6933	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6934	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6935	if (iSegReg == X86_SREG_SS)
6936	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6937	else
6938	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6939	return VINF_HM_PENDING_XCPT;
6940	}
6941	}
6942	}
6943	else
6944	{
6945	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6946	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6947	return VINF_HM_PENDING_XCPT;
6948	}
6949
6950	*pGCPtrMem = GCPtrMem;
6951	return VINF_SUCCESS;
6952	}
6953
6954
6955	/**
6956	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6957	* guest attempting to execute a VMX instruction.
6958	*
6959	* @returns Strict VBox status code (i.e. informational status codes too).
6960	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
6961	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
6962	*
6963	* @param pVCpu The cross context virtual CPU structure.
6964	* @param uExitReason The VM-exit reason.
6965	*
6966	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
6967	* @remarks No-long-jump zone!!!
6968	*/
6969	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
6970	{
6971	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
6972	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
6973
6974	if ( CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx)
6975	\|\| ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6976	&& !CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)))
6977	{
6978	Log4Func(("In real/v86-mode or long-mode outside 64-bit code segment -> #UD\n"));
6979	hmR0VmxSetPendingXcptUD(pVCpu);
6980	return VINF_HM_PENDING_XCPT;
6981	}
6982
6983	if (uExitReason == VMX_EXIT_VMXON)
6984	{
6985	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
6986
6987	/*
6988	* We check CR4.VMXE because it is required to be always set while in VMX operation
6989	* by physical CPUs and our CR4 read shadow is only consulted when executing specific
6990	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
6991	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
6992	*/
6993	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
6994	{
6995	Log4Func(("CR4.VMXE is not set -> #UD\n"));
6996	hmR0VmxSetPendingXcptUD(pVCpu);
6997	return VINF_HM_PENDING_XCPT;
6998	}
6999	}
7000	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
7001	{
7002	/*
7003	* The guest has not entered VMX operation but attempted to execute a VMX instruction
7004	* (other than VMXON), we need to raise a #UD.
7005	*/
7006	Log4Func(("Not in VMX root mode -> #UD\n"));
7007	hmR0VmxSetPendingXcptUD(pVCpu);
7008	return VINF_HM_PENDING_XCPT;
7009	}
7010
7011	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
7012	{
7013	/*
7014	* The nested-guest attempted to execute a VMX instruction, cause a VM-exit and let
7015	* the guest hypervisor deal with it.
7016	*/
7017	/** @todo NSTVMX: Trigger a VM-exit */
7018	}
7019
7020	/*
7021	* VMX instructions require CPL 0 except in VMX non-root mode where the VM-exit intercept
7022	* (above) takes preceedence over the CPL check.
7023	*/
7024	if (CPUMGetGuestCPL(pVCpu) > 0)
7025	{
7026	Log4Func(("CPL > 0 -> #GP(0)\n"));
7027	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7028	return VINF_HM_PENDING_XCPT;
7029	}
7030
7031	return VINF_SUCCESS;
7032	}
7033	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7034
7035
7036	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7037	{
7038	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7039
7040	/*
7041	* If VT-x marks the segment as unusable, most other bits remain undefined:
7042	* - For CS the L, D and G bits have meaning.
7043	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7044	* - For the remaining data segments no bits are defined.
7045	*
7046	* The present bit and the unusable bit has been observed to be set at the
7047	* same time (the selector was supposed to be invalid as we started executing
7048	* a V8086 interrupt in ring-0).
7049	*
7050	* What should be important for the rest of the VBox code, is that the P bit is
7051	* cleared. Some of the other VBox code recognizes the unusable bit, but
7052	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7053	* safe side here, we'll strip off P and other bits we don't care about. If
7054	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7055	*
7056	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7057	*/
7058	#ifdef VBOX_STRICT
7059	uint32_t const uAttr = pSelReg->Attr.u;
7060	#endif
7061
7062	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7063	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7064	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7065
7066	#ifdef VBOX_STRICT
7067	VMMRZCallRing3Disable(pVCpu);
7068	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7069	# ifdef DEBUG_bird
7070	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7071	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7072	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7073	# endif
7074	VMMRZCallRing3Enable(pVCpu);
7075	NOREF(uAttr);
7076	#endif
7077	RT_NOREF2(pVCpu, idxSel);
7078	}
7079
7080
7081	/**
7082	* Imports a guest segment register from the current VMCS into the guest-CPU
7083	* context.
7084	*
7085	* @returns VBox status code.
7086	* @param pVCpu The cross context virtual CPU structure.
7087	* @param iSegReg The segment register number (X86_SREG_XXX).
7088	*
7089	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7090	* do not log!
7091	*/
7092	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7093	{
7094	Assert(iSegReg < X86_SREG_COUNT);
7095
7096	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7097	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7098	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7099	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
7100	uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
7101	#else
7102	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7103	#endif
7104	uint64_t u64Base;
7105	uint32_t u32Sel, u32Limit, u32Attr;
7106	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7107	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7108	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7109	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
7110	if (RT_SUCCESS(rc))
7111	{
7112	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7113	pSelReg->Sel = u32Sel;
7114	pSelReg->ValidSel = u32Sel;
7115	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7116	pSelReg->u32Limit = u32Limit;
7117	pSelReg->u64Base = u64Base;
7118	pSelReg->Attr.u = u32Attr;
7119	if (u32Attr & X86DESCATTR_UNUSABLE)
7120	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7121	}
7122	return rc;
7123	}
7124
7125
7126	/**
7127	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7128	*
7129	* @returns VBox status code.
7130	* @param pVCpu The cross context virtual CPU structure.
7131	*
7132	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7133	* do not log!
7134	*/
7135	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7136	{
7137	uint64_t u64Base;
7138	uint32_t u32Sel, u32Limit, u32Attr;
7139	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7140	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7141	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7142	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7143	if (RT_SUCCESS(rc))
7144	{
7145	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7146	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7147	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7148	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7149	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7150	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7151	if (u32Attr & X86DESCATTR_UNUSABLE)
7152	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7153	}
7154	return rc;
7155	}
7156
7157
7158	/**
7159	* Imports the guest TR from the current VMCS into the guest-CPU context.
7160	*
7161	* @returns VBox status code.
7162	* @param pVCpu The cross context virtual CPU structure.
7163	*
7164	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7165	* do not log!
7166	*/
7167	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7168	{
7169	uint32_t u32Sel, u32Limit, u32Attr;
7170	uint64_t u64Base;
7171	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7172	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7173	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7174	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7175	AssertRCReturn(rc, rc);
7176
7177	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7178	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7179	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7180	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7181	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7182	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7183	/* TR is the only selector that can never be unusable. */
7184	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7185	return VINF_SUCCESS;
7186	}
7187
7188
7189	/**
7190	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7191	*
7192	* @returns VBox status code.
7193	* @param pVCpu The cross context virtual CPU structure.
7194	*
7195	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7196	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7197	* instead!!!
7198	*/
7199	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7200	{
7201	uint64_t u64Val;
7202	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7203	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7204	{
7205	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7206	if (RT_SUCCESS(rc))
7207	{
7208	pCtx->rip = u64Val;
7209	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7210	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7211	}
7212	return rc;
7213	}
7214	return VINF_SUCCESS;
7215	}
7216
7217
7218	/**
7219	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7220	*
7221	* @returns VBox status code.
7222	* @param pVCpu The cross context virtual CPU structure.
7223	* @param pVmcsInfo The VMCS info. object.
7224	*
7225	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7226	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7227	* instead!!!
7228	*/
7229	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7230	{
7231	uint32_t u32Val;
7232	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7233	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7234	{
7235	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7236	if (RT_SUCCESS(rc))
7237	{
7238	pCtx->eflags.u32 = u32Val;
7239
7240	/* Restore eflags for real-on-v86-mode hack. */
7241	if (pVmcsInfo->RealMode.fRealOnV86Active)
7242	{
7243	pCtx->eflags.Bits.u1VM = 0;
7244	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7245	}
7246	}
7247	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7248	return rc;
7249	}
7250	return VINF_SUCCESS;
7251	}
7252
7253
7254	/**
7255	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7256	* context.
7257	*
7258	* @returns VBox status code.
7259	* @param pVCpu The cross context virtual CPU structure.
7260	* @param pVmcsInfo The VMCS info. object.
7261	*
7262	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7263	* do not log!
7264	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7265	* instead!!!
7266	*/
7267	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7268	{
7269	uint32_t u32Val;
7270	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7271	if (RT_SUCCESS(rc))
7272	{
7273	if (!u32Val)
7274	{
7275	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7276	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7277
7278	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7279	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7280	}
7281	else
7282	{
7283	/*
7284	* We must import RIP here to set our EM interrupt-inhibited state.
7285	* We also import RFLAGS as our code that evaluates pending interrupts
7286	* before VM-entry requires it.
7287	*/
7288	rc = hmR0VmxImportGuestRip(pVCpu);
7289	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7290	if (RT_SUCCESS(rc))
7291	{
7292	if (u32Val & ( VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
7293	\| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7294	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7295	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7296	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7297
7298	if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
7299	{
7300	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7301	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
7302	}
7303	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7304	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7305	}
7306	}
7307	}
7308	return rc;
7309	}
7310
7311
7312	/**
7313	* Worker for VMXR0ImportStateOnDemand.
7314	*
7315	* @returns VBox status code.
7316	* @param pVCpu The cross context virtual CPU structure.
7317	* @param pVmcsInfo The VMCS info. object.
7318	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7319	*/
7320	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7321	{
7322	#define VMXLOCAL_BREAK_RC(a_rc) \
7323	if (RT_SUCCESS(a_rc)) \
7324	{ } \
7325	else \
7326	break
7327
7328	int rc = VINF_SUCCESS;
7329	PVM pVM = pVCpu->CTX_SUFF(pVM);
7330	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7331	uint64_t u64Val;
7332	uint32_t u32Val;
7333
7334	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
7335
7336	/*
7337	* We disable interrupts to make the updating of the state and in particular
7338	* the fExtrn modification atomic wrt to preemption hooks.
7339	*/
7340	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7341
7342	fWhat &= pCtx->fExtrn;
7343	if (fWhat)
7344	{
7345	do
7346	{
7347	if (fWhat & CPUMCTX_EXTRN_RIP)
7348	{
7349	rc = hmR0VmxImportGuestRip(pVCpu);
7350	VMXLOCAL_BREAK_RC(rc);
7351	}
7352
7353	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7354	{
7355	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7356	VMXLOCAL_BREAK_RC(rc);
7357	}
7358
7359	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7360	{
7361	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7362	VMXLOCAL_BREAK_RC(rc);
7363	}
7364
7365	if (fWhat & CPUMCTX_EXTRN_RSP)
7366	{
7367	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
7368	VMXLOCAL_BREAK_RC(rc);
7369	pCtx->rsp = u64Val;
7370	}
7371
7372	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7373	{
7374	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7375	if (fWhat & CPUMCTX_EXTRN_CS)
7376	{
7377	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7378	rc \|= hmR0VmxImportGuestRip(pVCpu);
7379	if (fRealOnV86Active)
7380	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7381	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7382	}
7383	if (fWhat & CPUMCTX_EXTRN_SS)
7384	{
7385	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7386	if (fRealOnV86Active)
7387	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7388	}
7389	if (fWhat & CPUMCTX_EXTRN_DS)
7390	{
7391	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7392	if (fRealOnV86Active)
7393	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7394	}
7395	if (fWhat & CPUMCTX_EXTRN_ES)
7396	{
7397	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7398	if (fRealOnV86Active)
7399	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7400	}
7401	if (fWhat & CPUMCTX_EXTRN_FS)
7402	{
7403	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7404	if (fRealOnV86Active)
7405	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7406	}
7407	if (fWhat & CPUMCTX_EXTRN_GS)
7408	{
7409	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7410	if (fRealOnV86Active)
7411	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7412	}
7413	VMXLOCAL_BREAK_RC(rc);
7414	}
7415
7416	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7417	{
7418	if (fWhat & CPUMCTX_EXTRN_LDTR)
7419	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
7420
7421	if (fWhat & CPUMCTX_EXTRN_GDTR)
7422	{
7423	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
7424	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
7425	pCtx->gdtr.pGdt = u64Val;
7426	pCtx->gdtr.cbGdt = u32Val;
7427	}
7428
7429	/* Guest IDTR. */
7430	if (fWhat & CPUMCTX_EXTRN_IDTR)
7431	{
7432	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
7433	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
7434	pCtx->idtr.pIdt = u64Val;
7435	pCtx->idtr.cbIdt = u32Val;
7436	}
7437
7438	/* Guest TR. */
7439	if (fWhat & CPUMCTX_EXTRN_TR)
7440	{
7441	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7442	don't need to import that one. */
7443	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7444	rc \|= hmR0VmxImportGuestTr(pVCpu);
7445	}
7446	VMXLOCAL_BREAK_RC(rc);
7447	}
7448
7449	if (fWhat & CPUMCTX_EXTRN_DR7)
7450	{
7451	if (!pVCpu->hm.s.fUsingHyperDR7)
7452	{
7453	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
7454	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
7455	VMXLOCAL_BREAK_RC(rc);
7456	pCtx->dr[7] = u32Val;
7457	}
7458	}
7459
7460	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7461	{
7462	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
7463	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
7464	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
7465	pCtx->SysEnter.cs = u32Val;
7466	VMXLOCAL_BREAK_RC(rc);
7467	}
7468
7469	#if HC_ARCH_BITS == 64
7470	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7471	{
7472	if ( pVM->hm.s.fAllow64BitGuests
7473	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7474	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7475	}
7476
7477	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7478	{
7479	if ( pVM->hm.s.fAllow64BitGuests
7480	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7481	{
7482	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7483	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7484	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7485	}
7486	}
7487	#endif
7488
7489	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7490	#if HC_ARCH_BITS == 32
7491	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
7492	#endif
7493	)
7494	{
7495	PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7496	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7497	Assert(cMsrs == 0 \|\| pMsr != NULL);
7498	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
7499	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
7500	{
7501	switch (pMsr->u32Msr)
7502	{
7503	#if HC_ARCH_BITS == 32
7504	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
7505	case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
7506	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
7507	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
7508	#endif
7509	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
7510	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
7511	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7512
7513	default:
7514	{
7515	pVCpu->hm.s.u32HMError = pMsr->u32Msr;
7516	ASMSetFlags(fEFlags);
7517	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
7518	cMsrs));
7519	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7520	}
7521	}
7522	}
7523	}
7524
7525	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7526	{
7527	uint64_t u64Shadow;
7528	if (fWhat & CPUMCTX_EXTRN_CR0)
7529	{
7530	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7531	* remove when we drop 32-bit host w/ 64-bit host support, see
7532	* @bugref{9180#c39}. */
7533	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7534	#if HC_ARCH_BITS == 32
7535	uint32_t u32Shadow;
7536	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
7537	u64Shadow = u32Shadow;
7538	#else
7539	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
7540	#endif
7541	VMXLOCAL_BREAK_RC(rc);
7542	u64Val = u32Val;
7543	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7544	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7545	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7546	CPUMSetGuestCR0(pVCpu, u64Val);
7547	VMMRZCallRing3Enable(pVCpu);
7548	}
7549
7550	if (fWhat & CPUMCTX_EXTRN_CR4)
7551	{
7552	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7553	* remove when we drop 32-bit host w/ 64-bit host support, see
7554	* @bugref{9180#c39}. */
7555	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7556	#if HC_ARCH_BITS == 32
7557	uint32_t u32Shadow;
7558	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
7559	u64Shadow = u32Shadow;
7560	#else
7561	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
7562	#endif
7563	VMXLOCAL_BREAK_RC(rc);
7564	u64Val = u32Val;
7565	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
7566	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7567	pCtx->cr4 = u64Val;
7568	}
7569
7570	if (fWhat & CPUMCTX_EXTRN_CR3)
7571	{
7572	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7573	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7574	\|\| ( pVM->hm.s.fNestedPaging
7575	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7576	{
7577	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
7578	if (pCtx->cr3 != u64Val)
7579	{
7580	pCtx->cr3 = u64Val;
7581	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7582	}
7583
7584	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7585	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7586	if (CPUMIsGuestInPAEModeEx(pCtx))
7587	{
7588	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
7589	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
7590	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
7591	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
7592	VMXLOCAL_BREAK_RC(rc);
7593	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7594	}
7595	}
7596	}
7597	}
7598	} while (0);
7599
7600	if (RT_SUCCESS(rc))
7601	{
7602	/* Update fExtrn. */
7603	pCtx->fExtrn &= ~fWhat;
7604
7605	/* If everything has been imported, clear the HM keeper bit. */
7606	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7607	{
7608	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7609	Assert(!pCtx->fExtrn);
7610	}
7611	}
7612	}
7613	else
7614	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7615
7616	ASMSetFlags(fEFlags);
7617
7618	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7619
7620	if (RT_SUCCESS(rc))
7621	{ /* likely */ }
7622	else
7623	return rc;
7624
7625	/*
7626	* Honor any pending CR3 updates.
7627	*
7628	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
7629	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7630	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7631	*
7632	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7633	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7634	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7635	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7636	*
7637	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7638	*/
7639	if (VMMRZCallRing3IsEnabled(pVCpu))
7640	{
7641	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7642	{
7643	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7644	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7645	}
7646
7647	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7648	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7649
7650	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7651	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7652	}
7653
7654	return VINF_SUCCESS;
7655	#undef VMXLOCAL_BREAK_RC
7656	}
7657
7658
7659	/**
7660	* Saves the guest state from the VMCS into the guest-CPU context.
7661	*
7662	* @returns VBox status code.
7663	* @param pVCpu The cross context virtual CPU structure.
7664	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7665	*/
7666	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
7667	{
7668	PCVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7669	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7670	}
7671
7672
7673	/**
7674	* Check per-VM and per-VCPU force flag actions that require us to go back to
7675	* ring-3 for one reason or another.
7676	*
7677	* @returns Strict VBox status code (i.e. informational status codes too)
7678	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7679	* ring-3.
7680	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7681	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7682	* interrupts)
7683	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7684	* all EMTs to be in ring-3.
7685	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7686	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7687	* to the EM loop.
7688	*
7689	* @param pVCpu The cross context virtual CPU structure.
7690	* @param fStepping Whether we are single-stepping the guest using the
7691	* hypervisor debugger.
7692	*/
7693	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
7694	{
7695	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7696
7697	/*
7698	* Update pending interrupts into the APIC's IRR.
7699	*/
7700	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7701	APICUpdatePendingInterrupts(pVCpu);
7702
7703	/*
7704	* Anything pending? Should be more likely than not if we're doing a good job.
7705	*/
7706	PVM pVM = pVCpu->CTX_SUFF(pVM);
7707	if ( !fStepping
7708	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7709	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7710	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7711	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7712	return VINF_SUCCESS;
7713
7714	/* Pending PGM C3 sync. */
7715	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7716	{
7717	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7718	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7719	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7720	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7721	if (rcStrict2 != VINF_SUCCESS)
7722	{
7723	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7724	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7725	return rcStrict2;
7726	}
7727	}
7728
7729	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7730	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7731	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7732	{
7733	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7734	int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7735	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7736	return rc2;
7737	}
7738
7739	/* Pending VM request packets, such as hardware interrupts. */
7740	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7741	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7742	{
7743	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7744	return VINF_EM_PENDING_REQUEST;
7745	}
7746
7747	/* Pending PGM pool flushes. */
7748	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7749	{
7750	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7751	return VINF_PGM_POOL_FLUSH_PENDING;
7752	}
7753
7754	/* Pending DMA requests. */
7755	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7756	{
7757	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7758	return VINF_EM_RAW_TO_R3;
7759	}
7760
7761	return VINF_SUCCESS;
7762	}
7763
7764
7765	/**
7766	* Converts any TRPM trap into a pending HM event. This is typically used when
7767	* entering from ring-3 (not longjmp returns).
7768	*
7769	* @param pVCpu The cross context virtual CPU structure.
7770	*/
7771	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7772	{
7773	Assert(TRPMHasTrap(pVCpu));
7774	Assert(!pVCpu->hm.s.Event.fPending);
7775
7776	uint8_t uVector;
7777	TRPMEVENT enmTrpmEvent;
7778	RTGCUINT uErrCode;
7779	RTGCUINTPTR GCPtrFaultAddress;
7780	uint8_t cbInstr;
7781
7782	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7783	AssertRC(rc);
7784
7785	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7786	uint32_t u32IntInfo = uVector \| VMX_EXIT_INT_INFO_VALID;
7787	if (enmTrpmEvent == TRPM_TRAP)
7788	{
7789	/** @todo r=ramshankar: TRPM currently offers no way to determine a \#DB that was
7790	* generated using INT1 (ICEBP). */
7791	switch (uVector)
7792	{
7793	case X86_XCPT_NMI:
7794	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7795	break;
7796
7797	case X86_XCPT_BP:
7798	case X86_XCPT_OF:
7799	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7800	break;
7801
7802	case X86_XCPT_PF:
7803	case X86_XCPT_DF:
7804	case X86_XCPT_TS:
7805	case X86_XCPT_NP:
7806	case X86_XCPT_SS:
7807	case X86_XCPT_GP:
7808	case X86_XCPT_AC:
7809	u32IntInfo \|= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7810	RT_FALL_THRU();
7811	default:
7812	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7813	break;
7814	}
7815	}
7816	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7817	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7818	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7819	{
7820	switch (uVector)
7821	{
7822	case X86_XCPT_BP:
7823	case X86_XCPT_OF:
7824	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7825	break;
7826
7827	default:
7828	Assert(uVector == X86_XCPT_DB);
7829	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7830	break;
7831	}
7832	}
7833	else
7834	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7835
7836	rc = TRPMResetTrap(pVCpu);
7837	AssertRC(rc);
7838	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7839	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7840
7841	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7842	}
7843
7844
7845	/**
7846	* Converts the pending HM event into a TRPM trap.
7847	*
7848	* @param pVCpu The cross context virtual CPU structure.
7849	*/
7850	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7851	{
7852	Assert(pVCpu->hm.s.Event.fPending);
7853
7854	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7855	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7856	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
7857	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7858
7859	/* If a trap was already pending, we did something wrong! */
7860	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7861
7862	/** @todo Use HMVmxEventToTrpmEventType() later. */
7863	TRPMEVENT enmTrapType;
7864	switch (uVectorType)
7865	{
7866	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7867	enmTrapType = TRPM_HARDWARE_INT;
7868	break;
7869
7870	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7871	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7872	enmTrapType = TRPM_TRAP;
7873	break;
7874
7875	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT: /* #DB (INT1/ICEBP). */
7876	Assert(uVector == X86_XCPT_DB);
7877	enmTrapType = TRPM_SOFTWARE_INT;
7878	break;
7879
7880	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP (INT3) and #OF (INTO) */
7881	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7882	enmTrapType = TRPM_SOFTWARE_INT;
7883	break;
7884
7885	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7886	enmTrapType = TRPM_SOFTWARE_INT;
7887	break;
7888
7889	default:
7890	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7891	enmTrapType = TRPM_32BIT_HACK;
7892	break;
7893	}
7894
7895	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7896
7897	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7898	AssertRC(rc);
7899
7900	if (fErrorCodeValid)
7901	TRPMSetErrorCode(pVCpu, uErrorCode);
7902
7903	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7904	&& uVector == X86_XCPT_PF)
7905	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7906	else if (enmTrapType == TRPM_SOFTWARE_INT)
7907	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7908
7909	/* We're now done converting the pending event. */
7910	pVCpu->hm.s.Event.fPending = false;
7911	}
7912
7913
7914	/**
7915	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7916	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7917	*
7918	* @param pVCpu The cross context virtual CPU structure.
7919	* @param pVmcsInfo The VMCS info. object.
7920	*/
7921	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
7922	{
7923	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7924	{
7925	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7926	{
7927	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7928	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7929	AssertRC(rc);
7930	}
7931	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
7932	}
7933
7934
7935	/**
7936	* Clears the interrupt-window exiting control in the VMCS.
7937	*
7938	* @param pVmcsInfo The VMCS info. object.
7939	*/
7940	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7941	{
7942	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7943	{
7944	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7945	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7946	}
7947	return VINF_SUCCESS;
7948	}
7949
7950
7951	/**
7952	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7953	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7954	*
7955	* @param pVCpu The cross context virtual CPU structure.
7956	* @param pVmcsInfo The VMCS info. object.
7957	*/
7958	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
7959	{
7960	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7961	{
7962	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7963	{
7964	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7965	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7966	AssertRC(rc);
7967	Log4Func(("Setup NMI-window exiting\n"));
7968	}
7969	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7970	}
7971
7972
7973	/**
7974	* Clears the NMI-window exiting control in the VMCS.
7975	*
7976	* @param pVmcsInfo The VMCS info. object.
7977	*/
7978	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7979	{
7980	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7981	{
7982	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7983	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7984	}
7985	return VINF_SUCCESS;
7986	}
7987
7988
7989	/**
7990	* Does the necessary state syncing before returning to ring-3 for any reason
7991	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7992	*
7993	* @returns VBox status code.
7994	* @param pVCpu The cross context virtual CPU structure.
7995	* @param fImportState Whether to import the guest state from the VMCS back
7996	* to the guest-CPU context.
7997	*
7998	* @remarks No-long-jmp zone!!!
7999	*/
8000	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
8001	{
8002	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8003	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8004
8005	RTCPUID idCpu = RTMpCpuId();
8006	Log4Func(("HostCpuId=%u\n", idCpu));
8007
8008	/*
8009	* !!! IMPORTANT !!!
8010	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
8011	*/
8012
8013	/* Save the guest state if necessary. */
8014	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8015	if (fImportState)
8016	{
8017	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8018	AssertRCReturn(rc, rc);
8019	}
8020
8021	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8022	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8023	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8024
8025	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8026	#ifdef VBOX_STRICT
8027	if (CPUMIsHyperDebugStateActive(pVCpu))
8028	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8029	#endif
8030	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8031	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8032	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8033
8034	#if HC_ARCH_BITS == 64
8035	/* Restore host-state bits that VT-x only restores partially. */
8036	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8037	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8038	{
8039	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8040	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8041	}
8042	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8043	#endif
8044
8045	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8046	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8047	{
8048	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8049	if (!fImportState)
8050	{
8051	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8052	AssertRCReturn(rc, rc);
8053	}
8054	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8055	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8056	}
8057	else
8058	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8059
8060	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8061	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8062
8063	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8064	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8065	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8066	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8067	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8068	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8069	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8070	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8071	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8072
8073	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8074
8075	/** @todo This partially defeats the purpose of having preemption hooks.
8076	* The problem is, deregistering the hooks should be moved to a place that
8077	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8078	* context.
8079	*/
8080	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8081	AssertRCReturn(rc, rc);
8082
8083	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8084	NOREF(idCpu);
8085	return VINF_SUCCESS;
8086	}
8087
8088
8089	/**
8090	* Leaves the VT-x session.
8091	*
8092	* @returns VBox status code.
8093	* @param pVCpu The cross context virtual CPU structure.
8094	*
8095	* @remarks No-long-jmp zone!!!
8096	*/
8097	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8098	{
8099	HM_DISABLE_PREEMPT(pVCpu);
8100	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8101	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8102	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8103
8104	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8105	and done this from the VMXR0ThreadCtxCallback(). */
8106	if (!pVCpu->hm.s.fLeaveDone)
8107	{
8108	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8109	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8110	pVCpu->hm.s.fLeaveDone = true;
8111	}
8112	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8113
8114	/*
8115	* !!! IMPORTANT !!!
8116	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8117	*/
8118
8119	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8120	/** @todo Deregistering here means we need to VMCLEAR always
8121	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8122	* for calling VMMR0ThreadCtxHookDisable here! */
8123	VMMR0ThreadCtxHookDisable(pVCpu);
8124
8125	/* Leave HM context. This takes care of local init (term). */
8126	int rc = HMR0LeaveCpu(pVCpu);
8127
8128	HM_RESTORE_PREEMPT();
8129	return rc;
8130	}
8131
8132
8133	/**
8134	* Does the necessary state syncing before doing a longjmp to ring-3.
8135	*
8136	* @returns VBox status code.
8137	* @param pVCpu The cross context virtual CPU structure.
8138	*
8139	* @remarks No-long-jmp zone!!!
8140	*/
8141	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8142	{
8143	return hmR0VmxLeaveSession(pVCpu);
8144	}
8145
8146
8147	/**
8148	* Take necessary actions before going back to ring-3.
8149	*
8150	* An action requires us to go back to ring-3. This function does the necessary
8151	* steps before we can safely return to ring-3. This is not the same as longjmps
8152	* to ring-3, this is voluntary and prepares the guest so it may continue
8153	* executing outside HM (recompiler/IEM).
8154	*
8155	* @returns VBox status code.
8156	* @param pVCpu The cross context virtual CPU structure.
8157	* @param rcExit The reason for exiting to ring-3. Can be
8158	* VINF_VMM_UNKNOWN_RING3_CALL.
8159	*/
8160	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8161	{
8162	Assert(pVCpu);
8163	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8164
8165	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8166	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8167	{
8168	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8169	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8170	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8171	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8172	}
8173
8174	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8175	VMMRZCallRing3Disable(pVCpu);
8176	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8177
8178	/*
8179	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8180	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8181	*
8182	* This is because execution may continue from ring-3 and we would need to inject
8183	* the event from there (hence place it back in TRPM).
8184	*/
8185	if (pVCpu->hm.s.Event.fPending)
8186	{
8187	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8188	Assert(!pVCpu->hm.s.Event.fPending);
8189
8190	/* Clear the events from the VMCS. */
8191	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8192	AssertRCReturn(rc, rc);
8193	}
8194	#ifdef VBOX_STRICT
8195	else
8196	{
8197	/*
8198	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8199	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8200	* occasionally, see @bugref{9180#c42}.
8201	*/
8202	uint32_t uEntryIntInfo;
8203	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8204	AssertRC(rc);
8205	Assert(!VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8206	}
8207	#endif
8208
8209	/*
8210	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8211	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8212	* (e.g. TPR below threshold).
8213	*/
8214	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8215	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8216	AssertRCReturn(rc, rc);
8217
8218	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8219	and if we're injecting an event we should have a TRPM trap pending. */
8220	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8221	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8222	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8223	#endif
8224
8225	/* Save guest state and restore host state bits. */
8226	rc = hmR0VmxLeaveSession(pVCpu);
8227	AssertRCReturn(rc, rc);
8228	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8229
8230	/* Thread-context hooks are unregistered at this point!!! */
8231
8232	/* Sync recompiler state. */
8233	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8234	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8235	\| CPUM_CHANGED_LDTR
8236	\| CPUM_CHANGED_GDTR
8237	\| CPUM_CHANGED_IDTR
8238	\| CPUM_CHANGED_TR
8239	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8240	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8241	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8242	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8243
8244	Assert(!pVCpu->hm.s.fClearTrapFlag);
8245
8246	/* Update the exit-to-ring 3 reason. */
8247	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8248
8249	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8250	if ( rcExit != VINF_EM_RAW_INTERRUPT
8251	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8252	{
8253	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8254	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8255	}
8256
8257	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8258
8259	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
8260	VMMRZCallRing3RemoveNotification(pVCpu);
8261	VMMRZCallRing3Enable(pVCpu);
8262
8263	return rc;
8264	}
8265
8266
8267	/**
8268	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8269	* longjump to ring-3 and possibly get preempted.
8270	*
8271	* @returns VBox status code.
8272	* @param pVCpu The cross context virtual CPU structure.
8273	* @param enmOperation The operation causing the ring-3 longjump.
8274	* @param pvUser User argument, currently unused, NULL.
8275	*/
8276	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
8277	{
8278	RT_NOREF(pvUser);
8279	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8280	{
8281	/*
8282	* !!! IMPORTANT !!!
8283	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8284	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8285	*/
8286	VMMRZCallRing3RemoveNotification(pVCpu);
8287	VMMRZCallRing3Disable(pVCpu);
8288	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
8289	RTThreadPreemptDisable(&PreemptState);
8290
8291	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8292	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8293	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8294	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8295
8296	#if HC_ARCH_BITS == 64
8297	/* Restore host-state bits that VT-x only restores partially. */
8298	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8299	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8300	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8301	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8302	#endif
8303
8304	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8305	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8306	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8307
8308	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8309	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8310	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8311
8312	/* Clear the current VMCS data back to memory. */
8313	hmR0VmxClearVmcs(pVmcsInfo);
8314
8315	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8316	VMMR0ThreadCtxHookDisable(pVCpu);
8317	HMR0LeaveCpu(pVCpu);
8318	RTThreadPreemptRestore(&PreemptState);
8319	return VINF_SUCCESS;
8320	}
8321
8322	Assert(pVCpu);
8323	Assert(pvUser);
8324	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8325	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8326
8327	VMMRZCallRing3Disable(pVCpu);
8328	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8329
8330	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8331
8332	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8333	AssertRCReturn(rc, rc);
8334
8335	VMMRZCallRing3Enable(pVCpu);
8336	return VINF_SUCCESS;
8337	}
8338
8339
8340	/**
8341	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8342	* stack.
8343	*
8344	* @returns Strict VBox status code (i.e. informational status codes too).
8345	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8346	* @param pVCpu The cross context virtual CPU structure.
8347	* @param uValue The value to push to the guest stack.
8348	*/
8349	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
8350	{
8351	/*
8352	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8353	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8354	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8355	*/
8356	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8357	if (pCtx->sp == 1)
8358	return VINF_EM_RESET;
8359	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8360	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8361	AssertRC(rc);
8362	return rc;
8363	}
8364
8365
8366	/**
8367	* Injects an event into the guest upon VM-entry by updating the relevant fields
8368	* in the VM-entry area in the VMCS.
8369	*
8370	* @returns Strict VBox status code (i.e. informational status codes too).
8371	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8372	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8373	*
8374	* @param pVCpu The cross context virtual CPU structure.
8375	* @param pVmxTransient The VMX-transient structure.
8376	* @param pEvent The event being injected.
8377	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state.
8378	* This will be updated if necessary. This cannot not
8379	* be NULL.
8380	* @param fStepping Whether we're single-stepping guest execution and
8381	* should return VINF_EM_DBG_STEPPED if the event is
8382	* injected directly (registers modified by us, not by
8383	* hardware on VM-entry).
8384	*/
8385	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8386	uint32_t *pfIntrState)
8387	{
8388	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8389	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8390	Assert(pfIntrState);
8391
8392	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8393	uint32_t u32IntInfo = pEvent->u64IntInfo;
8394	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8395	uint32_t const cbInstr = pEvent->cbInstr;
8396	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8397	uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8398	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8399
8400	#ifdef VBOX_STRICT
8401	/*
8402	* Validate the error-code-valid bit for hardware exceptions.
8403	* No error codes for exceptions in real-mode.
8404	*
8405	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8406	*/
8407	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8408	&& !CPUMIsGuestInRealModeEx(pCtx))
8409	{
8410	switch (uVector)
8411	{
8412	case X86_XCPT_PF:
8413	case X86_XCPT_DF:
8414	case X86_XCPT_TS:
8415	case X86_XCPT_NP:
8416	case X86_XCPT_SS:
8417	case X86_XCPT_GP:
8418	case X86_XCPT_AC:
8419	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8420	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8421	RT_FALL_THRU();
8422	default:
8423	break;
8424	}
8425	}
8426
8427	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8428	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8429	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8430	#endif
8431
8432	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8433
8434	/*
8435	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8436	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8437	* interrupt handler in the (real-mode) guest.
8438	*
8439	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8440	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8441	*/
8442	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8443	{
8444	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8445	{
8446	/*
8447	* For CPUs with unrestricted guest execution enabled and with the guest
8448	* in real-mode, we must not set the deliver-error-code bit.
8449	*
8450	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8451	*/
8452	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8453	}
8454	else
8455	{
8456	PVM pVM = pVCpu->CTX_SUFF(pVM);
8457	Assert(PDMVmmDevHeapIsEnabled(pVM));
8458	Assert(pVM->hm.s.vmx.pRealModeTSS);
8459	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8460
8461	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8462	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8463	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8464	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8465	AssertRCReturn(rc2, rc2);
8466
8467	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8468	size_t const cbIdtEntry = sizeof(X86IDTR16);
8469	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8470	{
8471	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8472	if (uVector == X86_XCPT_DF)
8473	return VINF_EM_RESET;
8474
8475	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8476	No error codes for exceptions in real-mode. */
8477	if (uVector == X86_XCPT_GP)
8478	{
8479	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8480	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8481	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8482	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8483	HMEVENT EventXcptDf;
8484	RT_ZERO(EventXcptDf);
8485	EventXcptDf.u64IntInfo = uXcptDfInfo;
8486	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8487	}
8488
8489	/*
8490	* If we're injecting an event with no valid IDT entry, inject a #GP.
8491	* No error codes for exceptions in real-mode.
8492	*
8493	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8494	*/
8495	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8496	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8497	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8498	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8499	HMEVENT EventXcptGp;
8500	RT_ZERO(EventXcptGp);
8501	EventXcptGp.u64IntInfo = uXcptGpInfo;
8502	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8503	}
8504
8505	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8506	uint16_t uGuestIp = pCtx->ip;
8507	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8508	{
8509	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8510	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8511	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8512	}
8513	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8514	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8515
8516	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8517	X86IDTR16 IdtEntry;
8518	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8519	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8520	AssertRCReturn(rc2, rc2);
8521
8522	/* Construct the stack frame for the interrupt/exception handler. */
8523	VBOXSTRICTRC rcStrict;
8524	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8525	if (rcStrict == VINF_SUCCESS)
8526	{
8527	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8528	if (rcStrict == VINF_SUCCESS)
8529	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8530	}
8531
8532	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8533	if (rcStrict == VINF_SUCCESS)
8534	{
8535	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8536	pCtx->rip = IdtEntry.offSel;
8537	pCtx->cs.Sel = IdtEntry.uSel;
8538	pCtx->cs.ValidSel = IdtEntry.uSel;
8539	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8540	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8541	&& uVector == X86_XCPT_PF)
8542	pCtx->cr2 = GCPtrFault;
8543
8544	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8545	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8546	\| HM_CHANGED_GUEST_RSP);
8547
8548	/*
8549	* If we delivered a hardware exception (other than an NMI) and if there was
8550	* block-by-STI in effect, we should clear it.
8551	*/
8552	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8553	{
8554	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8555	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8556	Log4Func(("Clearing inhibition due to STI\n"));
8557	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8558	}
8559
8560	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8561	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8562
8563	/*
8564	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8565	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8566	*/
8567	pVCpu->hm.s.Event.fPending = false;
8568
8569	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8570	if (fStepping)
8571	rcStrict = VINF_EM_DBG_STEPPED;
8572	}
8573	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8574	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8575	return rcStrict;
8576	}
8577	}
8578
8579	/*
8580	* Validate.
8581	*/
8582	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8583	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8584
8585	/*
8586	* Inject the event into the VMCS.
8587	*/
8588	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8589	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8590	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8591	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8592	AssertRCReturn(rc, rc);
8593
8594	/*
8595	* Update guest CR2 if this is a page-fault.
8596	*/
8597	if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8598	&& uVector == X86_XCPT_PF)
8599	pCtx->cr2 = GCPtrFault;
8600
8601	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8602	return VINF_SUCCESS;
8603	}
8604
8605
8606	/**
8607	* Evaluates the event to be delivered to the guest and sets it as the pending
8608	* event.
8609	*
8610	* @returns Strict VBox status code (i.e. informational status codes too).
8611	* @param pVCpu The cross context virtual CPU structure.
8612	* @param pVmxTransient The VMX-transient structure.
8613	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8614	*/
8615	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8616	{
8617	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8618	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8619
8620	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
8621	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmcsInfo);
8622	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8623	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8624	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8625
8626	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8627	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8628	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8629	Assert(!TRPMHasTrap(pVCpu));
8630	Assert(pfIntrState);
8631
8632	*pfIntrState = fIntrState;
8633
8634	/*
8635	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
8636	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
8637	*/
8638	/** @todo SMI. SMIs take priority over NMIs. */
8639	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
8640	{
8641	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8642	if ( !pVCpu->hm.s.Event.fPending
8643	&& !fBlockNmi
8644	&& !fBlockSti
8645	&& !fBlockMovSS)
8646	{
8647	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8648	if ( pVmxTransient->fIsNestedGuest
8649	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8650	return IEMExecVmxVmexitNmi(pVCpu);
8651	#endif
8652	hmR0VmxSetPendingXcptNmi(pVCpu);
8653	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8654	Log4Func(("Pending NMI\n"));
8655	}
8656	else
8657	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8658	}
8659	/*
8660	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
8661	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
8662	*/
8663	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8664	&& !pVCpu->hm.s.fSingleInstruction)
8665	{
8666	Assert(!DBGFIsStepping(pVCpu));
8667	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8668	AssertRCReturn(rc, rc);
8669	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8670	if ( !pVCpu->hm.s.Event.fPending
8671	&& !fBlockInt
8672	&& !fBlockSti
8673	&& !fBlockMovSS)
8674	{
8675	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8676	if ( pVmxTransient->fIsNestedGuest
8677	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
8678	{
8679	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector /, true / fIntPending */);
8680	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8681	return rcStrict;
8682	}
8683	#endif
8684	uint8_t u8Interrupt;
8685	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8686	if (RT_SUCCESS(rc))
8687	{
8688	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8689	if ( pVmxTransient->fIsNestedGuest
8690	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8691	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8692	{
8693	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8694	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8695	return rcStrict;
8696	}
8697	#endif
8698	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8699	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
8700	}
8701	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8702	{
8703	if ( !pVmxTransient->fIsNestedGuest
8704	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8705	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
8706	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8707
8708	/*
8709	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8710	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8711	* need to re-set this force-flag here.
8712	*/
8713	}
8714	else
8715	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8716	}
8717	else
8718	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8719	}
8720
8721	return VINF_SUCCESS;
8722	}
8723
8724
8725	/**
8726	* Injects any pending events into the guest if the guest is in a state to
8727	* receive them.
8728	*
8729	* @returns Strict VBox status code (i.e. informational status codes too).
8730	* @param pVCpu The cross context virtual CPU structure.
8731	* @param pVmxTransient The VMX-transient structure.
8732	* @param fIntrState The VT-x guest-interruptibility state.
8733	* @param fStepping Whether we are single-stepping the guest using the
8734	* hypervisor debugger and should return
8735	* VINF_EM_DBG_STEPPED if the event was dispatched
8736	* directly.
8737	*/
8738	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
8739	{
8740	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8741	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8742
8743	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8744	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8745
8746	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8747	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8748	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8749	Assert(!TRPMHasTrap(pVCpu));
8750
8751	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8752	if (pVCpu->hm.s.Event.fPending)
8753	{
8754	/*
8755	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8756	* pending even while injecting an event and in this case, we want a VM-exit as soon as
8757	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
8758	*
8759	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8760	*/
8761	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8762	#ifdef VBOX_STRICT
8763	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8764	{
8765	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
8766	Assert(!fBlockInt);
8767	Assert(!fBlockSti);
8768	Assert(!fBlockMovSS);
8769	}
8770	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
8771	{
8772	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8773	Assert(!fBlockSti);
8774	Assert(!fBlockMovSS);
8775	Assert(!fBlockNmi);
8776	}
8777	#endif
8778	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8779	uIntType));
8780
8781	/*
8782	* Inject the event and get any changes to the guest-interruptibility state.
8783	*
8784	* The guest-interruptibility state may need to be updated if we inject the event
8785	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
8786	*/
8787	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
8788	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8789
8790	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8791	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8792	else
8793	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8794	}
8795
8796	/*
8797	* Update the guest-interruptibility state.
8798	*
8799	* This is required for the real-on-v86 software interrupt injection case above, as well as
8800	* updates to the guest state from ring-3 or IEM/REM.
8801	*/
8802	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
8803	AssertRCReturn(rc, rc);
8804
8805	/*
8806	* There's no need to clear the VM-entry interruption-information field here if we're not
8807	* injecting anything. VT-x clears the valid bit on every VM-exit.
8808	*
8809	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8810	*/
8811
8812	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8813	NOREF(fBlockMovSS); NOREF(fBlockSti);
8814	return rcStrict;
8815	}
8816
8817
8818	/**
8819	* Enters the VT-x session.
8820	*
8821	* @returns VBox status code.
8822	* @param pVCpu The cross context virtual CPU structure.
8823	*/
8824	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
8825	{
8826	AssertPtr(pVCpu);
8827	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8828	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8829
8830	LogFlowFunc(("pVCpu=%p\n", pVCpu));
8831	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8832	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8833
8834	#ifdef VBOX_STRICT
8835	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8836	RTCCUINTREG uHostCR4 = ASMGetCR4();
8837	if (!(uHostCR4 & X86_CR4_VMXE))
8838	{
8839	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8840	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8841	}
8842	#endif
8843
8844	/*
8845	* Load the appropriate VMCS as the current and active one.
8846	*/
8847	PVMXVMCSINFO pVmcsInfo;
8848	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
8849	if (!fInNestedGuestMode)
8850	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
8851	else
8852	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
8853	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
8854	if (RT_SUCCESS(rc))
8855	{
8856	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
8857	pVCpu->hm.s.fLeaveDone = false;
8858	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8859
8860	/*
8861	* Do the EMT scheduled L1D flush here if needed.
8862	*/
8863	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8864	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8865	}
8866	return rc;
8867	}
8868
8869
8870	/**
8871	* The thread-context callback (only on platforms which support it).
8872	*
8873	* @param enmEvent The thread-context event.
8874	* @param pVCpu The cross context virtual CPU structure.
8875	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8876	* @thread EMT(pVCpu)
8877	*/
8878	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8879	{
8880	NOREF(fGlobalInit);
8881
8882	switch (enmEvent)
8883	{
8884	case RTTHREADCTXEVENT_OUT:
8885	{
8886	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8887	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8888	VMCPU_ASSERT_EMT(pVCpu);
8889
8890	/* No longjmps (logger flushes, locks) in this fragile context. */
8891	VMMRZCallRing3Disable(pVCpu);
8892	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8893
8894	/* Restore host-state (FPU, debug etc.) */
8895	if (!pVCpu->hm.s.fLeaveDone)
8896	{
8897	/*
8898	* Do -not- import the guest-state here as we might already be in the middle of importing
8899	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8900	*/
8901	hmR0VmxLeave(pVCpu, false /* fImportState */);
8902	pVCpu->hm.s.fLeaveDone = true;
8903	}
8904
8905	/* Leave HM context, takes care of local init (term). */
8906	int rc = HMR0LeaveCpu(pVCpu);
8907	AssertRC(rc);
8908
8909	/* Restore longjmp state. */
8910	VMMRZCallRing3Enable(pVCpu);
8911	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8912	break;
8913	}
8914
8915	case RTTHREADCTXEVENT_IN:
8916	{
8917	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8918	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8919	VMCPU_ASSERT_EMT(pVCpu);
8920
8921	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8922	VMMRZCallRing3Disable(pVCpu);
8923	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8924
8925	/* Initialize the bare minimum state required for HM. This takes care of
8926	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8927	int rc = hmR0EnterCpu(pVCpu);
8928	AssertRC(rc);
8929	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8930	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8931
8932	/* Load the active VMCS as the current one. */
8933	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8934	rc = hmR0VmxLoadVmcs(pVmcsInfo);
8935	AssertRC(rc);
8936	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8937	pVCpu->hm.s.fLeaveDone = false;
8938
8939	/* Do the EMT scheduled L1D flush if needed. */
8940	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8941	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8942
8943	/* Restore longjmp state. */
8944	VMMRZCallRing3Enable(pVCpu);
8945	break;
8946	}
8947
8948	default:
8949	break;
8950	}
8951	}
8952
8953
8954	/**
8955	* Exports the host state into the VMCS host-state area.
8956	* Sets up the VM-exit MSR-load area.
8957	*
8958	* The CPU state will be loaded from these fields on every successful VM-exit.
8959	*
8960	* @returns VBox status code.
8961	* @param pVCpu The cross context virtual CPU structure.
8962	*
8963	* @remarks No-long-jump zone!!!
8964	*/
8965	static int hmR0VmxExportHostState(PVMCPU pVCpu)
8966	{
8967	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8968
8969	int rc = VINF_SUCCESS;
8970	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8971	{
8972	rc = hmR0VmxExportHostControlRegs();
8973	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8974
8975	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8976	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8977
8978	rc = hmR0VmxExportHostMsrs(pVCpu);
8979	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8980
8981	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8982	}
8983	return rc;
8984	}
8985
8986
8987	/**
8988	* Saves the host state in the VMCS host-state.
8989	*
8990	* @returns VBox status code.
8991	* @param pVCpu The cross context virtual CPU structure.
8992	*
8993	* @remarks No-long-jump zone!!!
8994	*/
8995	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
8996	{
8997	AssertPtr(pVCpu);
8998	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8999
9000	/*
9001	* Export the host state here while entering HM context.
9002	* When thread-context hooks are used, we might get preempted and have to re-save the host
9003	* state but most of the time we won't be, so do it here before we disable interrupts.
9004	*/
9005	return hmR0VmxExportHostState(pVCpu);
9006	}
9007
9008
9009	/**
9010	* Exports the guest state into the VMCS guest-state area.
9011	*
9012	* The will typically be done before VM-entry when the guest-CPU state and the
9013	* VMCS state may potentially be out of sync.
9014	*
9015	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9016	* VM-entry controls.
9017	* Sets up the appropriate VMX non-root function to execute guest code based on
9018	* the guest CPU mode.
9019	*
9020	* @returns VBox strict status code.
9021	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9022	* without unrestricted guest execution and the VMMDev is not presently
9023	* mapped (e.g. EFI32).
9024	*
9025	* @param pVCpu The cross context virtual CPU structure.
9026	* @param pVmxTransient The VMX-transient structure.
9027	*
9028	* @remarks No-long-jump zone!!!
9029	*/
9030	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9031	{
9032	AssertPtr(pVCpu);
9033	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9034	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9035
9036	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9037
9038	/*
9039	* Determine real-on-v86 mode.
9040	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9041	*/
9042	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9043	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9044	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9045	pVmcsInfo->RealMode. fRealOnV86Active = false;
9046	else
9047	{
9048	Assert(!pVmxTransient->fIsNestedGuest);
9049	pVmcsInfo->RealMode.fRealOnV86Active = true;
9050	}
9051
9052	/*
9053	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9054	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9055	*/
9056	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9057	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9058	* be a need to evaluate this everytime since I'm pretty sure we intercept
9059	* all guest paging mode changes. */
9060	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9061	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9062
9063	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9064	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9065
9066	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9067	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9068
9069	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9070	if (rcStrict == VINF_SUCCESS)
9071	{ /* likely */ }
9072	else
9073	{
9074	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9075	return rcStrict;
9076	}
9077
9078	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9079	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9080
9081	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9082	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9083
9084	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9085	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9086
9087	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9088	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9089
9090	rc = hmR0VmxExportGuestRip(pVCpu);
9091	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9092	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9093	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9094
9095	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9096	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9097	\| HM_CHANGED_GUEST_CR2
9098	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9099	\| HM_CHANGED_GUEST_X87
9100	\| HM_CHANGED_GUEST_SSE_AVX
9101	\| HM_CHANGED_GUEST_OTHER_XSAVE
9102	\| HM_CHANGED_GUEST_XCRx
9103	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9104	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9105	\| HM_CHANGED_GUEST_TSC_AUX
9106	\| HM_CHANGED_GUEST_OTHER_MSRS
9107	\| HM_CHANGED_GUEST_HWVIRT /* More accurate PLE handling someday? */
9108	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9109
9110	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9111	return rc;
9112	}
9113
9114
9115	/**
9116	* Exports the state shared between the host and guest into the VMCS.
9117	*
9118	* @param pVCpu The cross context virtual CPU structure.
9119	* @param pVmxTransient The VMX-transient structure.
9120	*
9121	* @remarks No-long-jump zone!!!
9122	*/
9123	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9124	{
9125	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9126	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9127
9128	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9129	{
9130	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9131	AssertRC(rc);
9132	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9133
9134	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9135	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9136	{
9137	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9138	AssertRC(rc);
9139	}
9140	}
9141
9142	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9143	{
9144	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9145	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9146	}
9147
9148	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9149	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9150	}
9151
9152
9153	/**
9154	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9155	*
9156	* @returns Strict VBox status code (i.e. informational status codes too).
9157	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9158	* without unrestricted guest execution and the VMMDev is not presently
9159	* mapped (e.g. EFI32).
9160	*
9161	* @param pVCpu The cross context virtual CPU structure.
9162	* @param pVmxTransient The VMX-transient structure.
9163	*
9164	* @remarks No-long-jump zone!!!
9165	*/
9166	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9167	{
9168	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9169	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9170	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9171
9172	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9173	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9174	#endif
9175
9176	/*
9177	* For many exits it's only RIP that changes and hence try to export it first
9178	* without going through a lot of change flag checks.
9179	*/
9180	VBOXSTRICTRC rcStrict;
9181	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9182	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9183	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9184	{
9185	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9186	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9187	{ /* likely */}
9188	else
9189	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9190	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9191	}
9192	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9193	{
9194	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9195	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9196	{ /* likely */}
9197	else
9198	{
9199	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9200	VBOXSTRICTRC_VAL(rcStrict)));
9201	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9202	return rcStrict;
9203	}
9204	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9205	}
9206	else
9207	rcStrict = VINF_SUCCESS;
9208
9209	#ifdef VBOX_STRICT
9210	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9211	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9212	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9213	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9214	("fCtxChanged=%#RX64\n", fCtxChanged));
9215	#endif
9216	return rcStrict;
9217	}
9218
9219
9220	/**
9221	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9222	* and update error record fields accordingly.
9223	*
9224	* @return VMX_IGS_* return codes.
9225	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9226	* wrong with the guest state.
9227	*
9228	* @param pVCpu The cross context virtual CPU structure.
9229	* @param pVmcsInfo The VMCS info. object.
9230	*
9231	* @remarks This function assumes our cache of the VMCS controls
9232	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9233	*/
9234	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
9235	{
9236	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9237	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9238	uError = (err); \
9239	break; \
9240	} else do { } while (0)
9241
9242	int rc;
9243	PVM pVM = pVCpu->CTX_SUFF(pVM);
9244	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9245	uint32_t uError = VMX_IGS_ERROR;
9246	uint32_t u32Val;
9247	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9248
9249	do
9250	{
9251	/*
9252	* CR0.
9253	*/
9254	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9255	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9256	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
9257	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9258	if (fUnrestrictedGuest)
9259	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
9260
9261	uint32_t u32GuestCr0;
9262	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
9263	AssertRCBreak(rc);
9264	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9265	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9266	if ( !fUnrestrictedGuest
9267	&& (u32GuestCr0 & X86_CR0_PG)
9268	&& !(u32GuestCr0 & X86_CR0_PE))
9269	{
9270	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9271	}
9272
9273	/*
9274	* CR4.
9275	*/
9276	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9277	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9278
9279	uint32_t u32GuestCr4;
9280	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
9281	AssertRCBreak(rc);
9282	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9283	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9284
9285	/*
9286	* IA32_DEBUGCTL MSR.
9287	*/
9288	uint64_t u64Val;
9289	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9290	AssertRCBreak(rc);
9291	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9292	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9293	{
9294	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9295	}
9296	uint64_t u64DebugCtlMsr = u64Val;
9297
9298	#ifdef VBOX_STRICT
9299	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9300	AssertRCBreak(rc);
9301	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9302	#endif
9303	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9304
9305	/*
9306	* RIP and RFLAGS.
9307	*/
9308	uint32_t u32Eflags;
9309	#if HC_ARCH_BITS == 64
9310	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9311	AssertRCBreak(rc);
9312	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9313	if ( !fLongModeGuest
9314	\|\| !pCtx->cs.Attr.n.u1Long)
9315	{
9316	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9317	}
9318	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9319	* must be identical if the "IA-32e mode guest" VM-entry
9320	* control is 1 and CS.L is 1. No check applies if the
9321	* CPU supports 64 linear-address bits. */
9322
9323	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9324	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9325	AssertRCBreak(rc);
9326	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9327	VMX_IGS_RFLAGS_RESERVED);
9328	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9329	u32Eflags = u64Val;
9330	#else
9331	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9332	AssertRCBreak(rc);
9333	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9334	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9335	#endif
9336
9337	if ( fLongModeGuest
9338	\|\| ( fUnrestrictedGuest
9339	&& !(u32GuestCr0 & X86_CR0_PE)))
9340	{
9341	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9342	}
9343
9344	uint32_t u32EntryInfo;
9345	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9346	AssertRCBreak(rc);
9347	if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9348	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9349	{
9350	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9351	}
9352
9353	/*
9354	* 64-bit checks.
9355	*/
9356	#if HC_ARCH_BITS == 64
9357	if (fLongModeGuest)
9358	{
9359	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9360	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9361	}
9362
9363	if ( !fLongModeGuest
9364	&& (u32GuestCr4 & X86_CR4_PCIDE))
9365	{
9366	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9367	}
9368
9369	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9370	* 51:32 beyond the processor's physical-address width are 0. */
9371
9372	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9373	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9374	{
9375	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9376	}
9377
9378	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9379	AssertRCBreak(rc);
9380	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9381
9382	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9383	AssertRCBreak(rc);
9384	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9385	#endif
9386
9387	/*
9388	* PERF_GLOBAL MSR.
9389	*/
9390	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9391	{
9392	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9393	AssertRCBreak(rc);
9394	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9395	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9396	}
9397
9398	/*
9399	* PAT MSR.
9400	*/
9401	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9402	{
9403	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9404	AssertRCBreak(rc);
9405	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9406	for (unsigned i = 0; i < 8; i++)
9407	{
9408	uint8_t u8Val = (u64Val & 0xff);
9409	if ( u8Val != 0 /* UC */
9410	&& u8Val != 1 /* WC */
9411	&& u8Val != 4 /* WT */
9412	&& u8Val != 5 /* WP */
9413	&& u8Val != 6 /* WB */
9414	&& u8Val != 7 /* UC- */)
9415	{
9416	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9417	}
9418	u64Val >>= 8;
9419	}
9420	}
9421
9422	/*
9423	* EFER MSR.
9424	*/
9425	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9426	{
9427	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9428	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9429	AssertRCBreak(rc);
9430	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9431	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9432	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9433	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9434	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9435	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9436	* iemVmxVmentryCheckGuestState(). */
9437	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9438	\|\| !(u32GuestCr0 & X86_CR0_PG)
9439	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9440	VMX_IGS_EFER_LMA_LME_MISMATCH);
9441	}
9442
9443	/*
9444	* Segment registers.
9445	*/
9446	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9447	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9448	if (!(u32Eflags & X86_EFL_VM))
9449	{
9450	/* CS */
9451	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9452	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9453	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9454	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9455	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9456	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9457	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9458	/* CS cannot be loaded with NULL in protected mode. */
9459	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9460	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9461	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9462	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9463	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9464	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9465	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9466	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9467	else
9468	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9469
9470	/* SS */
9471	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9472	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9473	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9474	if ( !(pCtx->cr0 & X86_CR0_PE)
9475	\|\| pCtx->cs.Attr.n.u4Type == 3)
9476	{
9477	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9478	}
9479	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9480	{
9481	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9482	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9483	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9484	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9485	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9486	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9487	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9488	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9489	}
9490
9491	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9492	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9493	{
9494	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9495	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9496	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9497	\|\| pCtx->ds.Attr.n.u4Type > 11
9498	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9499	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9500	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9501	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9502	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9503	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9504	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9505	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9506	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9507	}
9508	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9509	{
9510	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9511	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9512	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9513	\|\| pCtx->es.Attr.n.u4Type > 11
9514	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9515	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9516	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9517	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9518	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9519	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9520	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9521	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9522	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9523	}
9524	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9525	{
9526	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9527	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9528	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9529	\|\| pCtx->fs.Attr.n.u4Type > 11
9530	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9531	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9532	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9533	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9534	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9535	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9536	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9537	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9538	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9539	}
9540	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9541	{
9542	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9543	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9544	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9545	\|\| pCtx->gs.Attr.n.u4Type > 11
9546	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9547	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9548	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9549	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9550	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9551	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9552	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9553	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9554	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9555	}
9556	/* 64-bit capable CPUs. */
9557	#if HC_ARCH_BITS == 64
9558	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9559	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9560	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9561	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9562	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9563	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9564	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9565	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9566	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9567	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9568	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9569	#endif
9570	}
9571	else
9572	{
9573	/* V86 mode checks. */
9574	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9575	if (pVmcsInfo->RealMode.fRealOnV86Active)
9576	{
9577	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9578	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9579	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9580	}
9581	else
9582	{
9583	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9584	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9585	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9586	}
9587
9588	/* CS */
9589	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9590	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9591	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9592	/* SS */
9593	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9594	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9595	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9596	/* DS */
9597	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9598	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9599	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9600	/* ES */
9601	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9602	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9603	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9604	/* FS */
9605	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9606	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9607	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9608	/* GS */
9609	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9610	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9611	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9612	/* 64-bit capable CPUs. */
9613	#if HC_ARCH_BITS == 64
9614	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9615	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9616	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9617	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9618	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9619	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9620	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9621	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9622	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9623	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9624	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9625	#endif
9626	}
9627
9628	/*
9629	* TR.
9630	*/
9631	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9632	/* 64-bit capable CPUs. */
9633	#if HC_ARCH_BITS == 64
9634	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9635	#endif
9636	if (fLongModeGuest)
9637	{
9638	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9639	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9640	}
9641	else
9642	{
9643	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9644	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9645	VMX_IGS_TR_ATTR_TYPE_INVALID);
9646	}
9647	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9648	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9649	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9650	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9651	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9652	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9653	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9654	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9655
9656	/*
9657	* GDTR and IDTR.
9658	*/
9659	#if HC_ARCH_BITS == 64
9660	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9661	AssertRCBreak(rc);
9662	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9663
9664	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9665	AssertRCBreak(rc);
9666	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9667	#endif
9668
9669	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9670	AssertRCBreak(rc);
9671	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9672
9673	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9674	AssertRCBreak(rc);
9675	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9676
9677	/*
9678	* Guest Non-Register State.
9679	*/
9680	/* Activity State. */
9681	uint32_t u32ActivityState;
9682	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9683	AssertRCBreak(rc);
9684	HMVMX_CHECK_BREAK( !u32ActivityState
9685	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9686	VMX_IGS_ACTIVITY_STATE_INVALID);
9687	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9688	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9689	uint32_t u32IntrState;
9690	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9691	AssertRCBreak(rc);
9692	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9693	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9694	{
9695	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9696	}
9697
9698	/** @todo Activity state and injecting interrupts. Left as a todo since we
9699	* currently don't use activity states but ACTIVE. */
9700
9701	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9702	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9703
9704	/* Guest interruptibility-state. */
9705	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9706	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9707	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9708	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9709	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9710	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9711	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9712	if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
9713	{
9714	if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9715	{
9716	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9717	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9718	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9719	}
9720	else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9721	{
9722	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9723	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9724	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9725	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9726	}
9727	}
9728	/** @todo Assumes the processor is not in SMM. */
9729	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9730	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9731	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9732	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9733	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9734	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9735	&& VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9736	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9737	{
9738	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
9739	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9740	}
9741
9742	/* Pending debug exceptions. */
9743	#if HC_ARCH_BITS == 64
9744	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
9745	AssertRCBreak(rc);
9746	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9747	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9748	u32Val = u64Val; /* For pending debug exceptions checks below. */
9749	#else
9750	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
9751	AssertRCBreak(rc);
9752	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9753	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9754	#endif
9755
9756	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9757	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
9758	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9759	{
9760	if ( (u32Eflags & X86_EFL_TF)
9761	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9762	{
9763	/* Bit 14 is PendingDebug.BS. */
9764	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9765	}
9766	if ( !(u32Eflags & X86_EFL_TF)
9767	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9768	{
9769	/* Bit 14 is PendingDebug.BS. */
9770	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9771	}
9772	}
9773
9774	/* VMCS link pointer. */
9775	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9776	AssertRCBreak(rc);
9777	if (u64Val != UINT64_C(0xffffffffffffffff))
9778	{
9779	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9780	/** @todo Bits beyond the processor's physical-address width MBZ. */
9781	/** @todo 32-bit located in memory referenced by value of this field (as a
9782	* physical address) must contain the processor's VMCS revision ID. */
9783	/** @todo SMM checks. */
9784	}
9785
9786	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9787	* not using nested paging? */
9788	if ( pVM->hm.s.fNestedPaging
9789	&& !fLongModeGuest
9790	&& CPUMIsGuestInPAEModeEx(pCtx))
9791	{
9792	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9793	AssertRCBreak(rc);
9794	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9795
9796	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9797	AssertRCBreak(rc);
9798	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9799
9800	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9801	AssertRCBreak(rc);
9802	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9803
9804	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9805	AssertRCBreak(rc);
9806	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9807	}
9808
9809	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9810	if (uError == VMX_IGS_ERROR)
9811	uError = VMX_IGS_REASON_NOT_FOUND;
9812	} while (0);
9813
9814	pVCpu->hm.s.u32HMError = uError;
9815	return uError;
9816
9817	#undef HMVMX_ERROR_BREAK
9818	#undef HMVMX_CHECK_BREAK
9819	}
9820
9821
9822	/**
9823	* Setup the APIC-access page for virtualizing APIC access.
9824	*
9825	* This can cause a longjumps to R3 due to the acquisition of the PGM lock, hence
9826	* this not done as part of exporting guest state, see @bugref{8721}.
9827	*
9828	* @returns VBox status code.
9829	* @param pVCpu The cross context virtual CPU structure.
9830	*/
9831	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
9832	{
9833	PVM pVM = pVCpu->CTX_SUFF(pVM);
9834	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
9835
9836	Assert(PDMHasApic(pVM));
9837	Assert(u64MsrApicBase);
9838
9839	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
9840	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
9841
9842	/* Unalias any existing mapping. */
9843	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
9844	AssertRCReturn(rc, rc);
9845
9846	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
9847	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
9848	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
9849	AssertRCReturn(rc, rc);
9850
9851	/* Update the per-VCPU cache of the APIC base MSR. */
9852	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
9853	return VINF_SUCCESS;
9854	}
9855
9856
9857	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9858	/**
9859	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
9860	* nested-guest using hardware-assisted VMX.
9861	*
9862	* @param pVCpu The cross context virtual CPU structure.
9863	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
9864	* @param pVmcsInfoGst The guest VMCS info. object.
9865	*/
9866	static void hmR0VmxMergeMsrBitmapNested(PCVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
9867	{
9868	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
9869	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
9870	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
9871	Assert(pu64MsrBitmapNstGst);
9872	Assert(pu64MsrBitmapGst);
9873	Assert(pu64MsrBitmap);
9874
9875	/*
9876	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
9877	* MSR that is intercepted by the guest is also intercepted while executing the
9878	* nested-guest using hardware-assisted VMX.
9879	*/
9880	uint32_t const cbFrag = sizeof(uint64_t);
9881	uint32_t const cFrags = X86_PAGE_4K_SIZE / cbFrag;
9882	for (uint32_t i = 0; i <= cFrags; i++)
9883	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
9884	}
9885
9886
9887	/**
9888	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
9889	* hardware-assisted VMX execution of the nested-guest.
9890	*
9891	* For a guest, we don't modify these controls once we set up the VMCS.
9892	*
9893	* For nested-guests since the guest hypervisor provides these controls on every
9894	* nested-guest VM-entry and could potentially change them everytime we need to
9895	* merge them before every nested-guest VM-entry.
9896	*
9897	* @returns VBox status code.
9898	* @param pVCpu The cross context virtual CPU structure.
9899	*/
9900	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
9901	{
9902	PVM pVM = pVCpu->CTX_SUFF(pVM);
9903	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
9904	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
9905	Assert(pVmcsNstGst);
9906
9907	/*
9908	* Merge the controls with the requirements of the guest VMCS.
9909	*
9910	* We do not need to validate the nested-guest VMX features specified in the
9911	* nested-guest VMCS with the features supported by the physical CPU as it's
9912	* already done by the VMLAUNCH/VMRESUME instruction emulation.
9913	*
9914	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the
9915	* guest are derived from the VMX features supported by the physical CPU.
9916	*/
9917
9918	/* Pin-based VM-execution controls. */
9919	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
9920
9921	/* Processor-based VM-execution controls. */
9922	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
9923	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
9924	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
9925	\| VMX_PROC_CTLS_USE_TPR_SHADOW
9926	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
9927
9928	/* Secondary processor-based VM-execution controls. */
9929	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
9930	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
9931	\| VMX_PROC_CTLS2_INVPCID
9932	\| VMX_PROC_CTLS2_RDTSCP
9933	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
9934	\| VMX_PROC_CTLS2_APIC_REG_VIRT
9935	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
9936	\| VMX_PROC_CTLS2_VMFUNC));
9937
9938	/*
9939	* VM-entry controls:
9940	* These controls contains state that depends on the nested-guest state (primarily
9941	* EFER MSR) and is thus not constant through VMLAUNCH/VMRESUME and the nested-guest
9942	* VM-exit. Although the nested-hypervisor cannot change it, we need to in order to
9943	* properly continue executing the nested-guest if the EFER MSR changes but does not
9944	* cause a nested-guest VM-exits.
9945	*
9946	* VM-exit controls:
9947	* These controls specify the host state on return. We cannot use the controls from
9948	* the nested-hypervisor state as is as it would contain the guest state rather than
9949	* the host state. Since the host state is subject to change (e.g. preemption, trips
9950	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
9951	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
9952	*
9953	* VM-entry MSR-load:
9954	* The guest MSRs from the VM-entry MSR-load area are already loaded into the
9955	* guest-CPU context by the VMLAUNCH/VMRESUME instruction emulation.
9956	*
9957	* VM-exit MSR-store:
9958	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU
9959	* context back into the VM-exit MSR-store area.
9960	*
9961	* VM-exit MSR-load areas:
9962	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence,
9963	* we can entirely ignore what the nested-hypervisor wants to load here.
9964	*/
9965
9966	/*
9967	* Exception bitmap.
9968	*
9969	* We could remove #UD from the guest bitmap and merge it with the nested-guest
9970	* bitmap here (and avoid doing anything while exporting nested-guest state), but to
9971	* keep the code more flexible if intercepting exceptions become more dynamic in
9972	* the future we do it as part of exporting the nested-guest state.
9973	*/
9974	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
9975
9976	/*
9977	* CR0/CR4 guest/host mask.
9978	*
9979	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest
9980	* must cause VM-exits, so we need to merge them here.
9981	*/
9982	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
9983	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
9984
9985	/*
9986	* Page-fault error-code mask and match.
9987	*
9988	* Although we require unrestricted guest execution (and thereby nested-paging) for
9989	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
9990	* normally intercept #PFs, it might intercept them for debugging purposes.
9991	*
9992	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF
9993	* filters. If the outer guest is intercepting #PFs we must intercept all #PFs.
9994	*/
9995	uint32_t u32XcptPFMask;
9996	uint32_t u32XcptPFMatch;
9997	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
9998	{
9999	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10000	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10001	}
10002	else
10003	{
10004	u32XcptPFMask = 0;
10005	u32XcptPFMatch = 0;
10006	}
10007
10008	/*
10009	* Pause-Loop exiting.
10010	*/
10011	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10012	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10013
10014	/*
10015	* I/O Bitmap.
10016	*
10017	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we
10018	* always intercept all I/O port accesses.
10019	*/
10020	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10021
10022	/*
10023	* APIC-access page.
10024	*
10025	* The APIC-access page address has already been initialized while setting up the
10026	* nested-guest VMCS. In theory, even if the guest-physical address is invalid, it
10027	* should not be on any consequence to the host or to the guest for that matter, but
10028	* we only accept valid addresses verified by the VMLAUNCH/VMRESUME instruction
10029	* emulation to keep it simple.
10030	*/
10031
10032	/*
10033	* Virtual-APIC page and TPR threshold.
10034	*
10035	* We shall use the host-physical address of the virtual-APIC page in guest memory directly.
10036	* For this reason, we can access the virtual-APIC page of the nested-guest only using
10037	* PGM physical handlers as we must not assume a kernel virtual-address mapping exists and
10038	* requesting PGM for a mapping could be expensive/resource intensive (PGM mapping cache).
10039	*/
10040	RTHCPHYS HCPhysVirtApic = NIL_RTHCPHYS;
10041	uint32_t const u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10042	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10043	{
10044	int rc = PGMPhysGCPhys2HCPhys(pVM, pVmcsNstGst->u64AddrVirtApic.u, &HCPhysVirtApic);
10045
10046	/*
10047	* If the guest hypervisor has loaded crap into the virtual-APIC page field
10048	* we would fail to obtain a valid host-physical address for its guest-physical
10049	* address.
10050	*
10051	* We currently do not support this scenario. Maybe in the future if there is a
10052	* pressing need we can explore making this particular set of conditions work.
10053	* Right now we just cause a VM-entry failure.
10054	*
10055	* This has already been checked by VMLAUNCH/VMRESUME instruction emulation,
10056	* so should not really failure at the moment.
10057	*/
10058	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10059	}
10060	else
10061	{
10062	/*
10063	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10064	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10065	* be taken care of by EPT/shadow paging.
10066	*/
10067	if (pVM->hm.s.fAllow64BitGuests)
10068	{
10069	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10070	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10071	}
10072	}
10073
10074	/*
10075	* Validate basic assumptions.
10076	*/
10077	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10078	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10079	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10080	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10081
10082	/*
10083	* Commit it to the nested-guest VMCS.
10084	*/
10085	int rc = VINF_SUCCESS;
10086	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10087	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10088	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10089	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10090	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10091	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10092	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10093	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10094	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10095	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10096	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10097	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10098	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10099	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10100	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10101	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10102	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10103	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10104	{
10105	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10106	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10107	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10108	}
10109	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10110	{
10111	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10112	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10113	}
10114	AssertRCReturn(rc, rc);
10115
10116	/*
10117	* Update the nested-guest VMCS cache.
10118	*/
10119	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10120	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10121	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10122	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10123	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10124	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10125	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10126	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10127	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10128
10129	/*
10130	* MSR bitmap.
10131	*
10132	* The MSR bitmap address has already been initialized while setting up the
10133	* nested-guest VMCS, here we need to merge the MSR bitmaps.
10134	*/
10135	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10136	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10137
10138	return VINF_SUCCESS;
10139	}
10140	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10141
10142
10143	/**
10144	* Does the preparations before executing guest code in VT-x.
10145	*
10146	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10147	* recompiler/IEM. We must be cautious what we do here regarding committing
10148	* guest-state information into the VMCS assuming we assuredly execute the
10149	* guest in VT-x mode.
10150	*
10151	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10152	* the common-state (TRPM/forceflags), we must undo those changes so that the
10153	* recompiler/IEM can (and should) use them when it resumes guest execution.
10154	* Otherwise such operations must be done when we can no longer exit to ring-3.
10155	*
10156	* @returns Strict VBox status code (i.e. informational status codes too).
10157	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10158	* have been disabled.
10159	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10160	* double-fault into the guest.
10161	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10162	* dispatched directly.
10163	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10164	*
10165	* @param pVCpu The cross context virtual CPU structure.
10166	* @param pVmxTransient The VMX-transient structure.
10167	* @param fStepping Whether we are single-stepping the guest in the
10168	* hypervisor debugger. Makes us ignore some of the reasons
10169	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10170	* if event dispatching took place.
10171	*/
10172	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10173	{
10174	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10175
10176	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10177	if (pVmxTransient->fIsNestedGuest)
10178	{
10179	RT_NOREF2(pVCpu, fStepping);
10180	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10181	return VINF_EM_RESCHEDULE_REM;
10182	}
10183	#endif
10184
10185	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10186	PGMRZDynMapFlushAutoSet(pVCpu);
10187	#endif
10188
10189	/*
10190	* Check and process force flag actions, some of which might require us to go back to ring-3.
10191	*/
10192	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10193	if (rcStrict == VINF_SUCCESS)
10194	{ /* FFs don't get set all the time. */ }
10195	else
10196	return rcStrict;
10197
10198	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10199	/*
10200	* Switch to the nested-guest VMCS as we may have transitioned into executing
10201	* the nested-guest without leaving ring-0. Otherwise, if we came from ring-3
10202	* we would load the nested-guest VMCS while entering the VMX ring-0 session.
10203	*
10204	* We do this as late as possible to minimize (though not completely remove)
10205	* clearing/loading VMCS again due to premature trips to ring-3 above.
10206	*/
10207	if (pVmxTransient->fIsNestedGuest)
10208	{
10209	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10210	{
10211	/*
10212	* Ensure we have synced everything from the guest VMCS and also flag that
10213	* that we need to export the full (nested) guest-CPU context to the
10214	* nested-guest VMCS.
10215	*/
10216	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
10217	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST);
10218
10219	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
10220	int rc = hmR0VmxSwitchVmcs(&pVCpu->hm.s.vmx.VmcsInfo, &pVCpu->hm.s.vmx.VmcsInfoNstGst);
10221	if (RT_LIKELY(rc == VINF_SUCCESS))
10222	{
10223	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = true;
10224	ASMSetFlags(fEFlags);
10225	pVmxTransient->pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10226
10227	/*
10228	* We use a different VM-exit MSR-store area for the nested-guest. Hence,
10229	* flag that we need to update the host MSR values there. Even if we decide
10230	* in the future to share the VM-exit MSR-store area page with the guest,
10231	* if its content differs, we would have to update the host MSRs anyway.
10232	*/
10233	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
10234	Assert(!pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer); /** @todo NSTVMX: Paranoia remove later. */
10235	}
10236	else
10237	{
10238	ASMSetFlags(fEFlags);
10239	return rc;
10240	}
10241	}
10242
10243	/*
10244	* Merge guest VMCS controls with the nested-guest VMCS controls.
10245	*
10246	* Even if we have not executed the guest prior to this (e.g. when resuming
10247	* from a saved state), we should be okay with merging controls as we
10248	* initialize the guest VMCS controls as part of VM setup phase.
10249	*/
10250	if (!pVCpu->hm.s.vmx.fMergedNstGstCtls)
10251	{
10252	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10253	AssertRCReturn(rc, rc);
10254	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10255	}
10256	}
10257	#endif
10258
10259	/*
10260	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10261	* We look at the guest VMCS control here as we always set it when supported by
10262	* the physical CPU. Looking at the nested-guest control here would not be
10263	* possible because they are not merged yet.
10264	*/
10265	PVM pVM = pVCpu->CTX_SUFF(pVM);
10266	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10267	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10268	&& (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10269	&& PDMHasApic(pVM))
10270	{
10271	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10272	AssertRCReturn(rc, rc);
10273	}
10274
10275	/*
10276	* Evaluate events to be injected into the guest.
10277	*
10278	* Events in TRPM can be injected without inspecting the guest state.
10279	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10280	* guest to cause a VM-exit the next time they are ready to receive the event.
10281	*/
10282	if (TRPMHasTrap(pVCpu))
10283	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10284
10285	uint32_t fIntrState;
10286	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10287
10288	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10289	/*
10290	* While evaluating pending events if something failed (unlikely) or if we were
10291	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10292	*/
10293	if ( rcStrict != VINF_SUCCESS
10294	\|\| ( pVmxTransient->fIsNestedGuest
10295	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)))
10296	return rcStrict;
10297	#endif
10298
10299	/*
10300	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10301	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10302	* also result in triple-faulting the VM.
10303	*
10304	* The above does not apply when executing a nested-guest (since unrestricted guest execution
10305	* is a requirement) regardless doing it avoid duplicating code elsewhere.
10306	*/
10307	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10308	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10309	{ /* likely */ }
10310	else
10311	{
10312	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10313	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10314	return rcStrict;
10315	}
10316
10317	/*
10318	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10319	* import CR3 themselves. We will need to update them here, as even as late as the above
10320	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10321	* the below force flags to be set.
10322	*/
10323	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10324	{
10325	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10326	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10327	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10328	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10329	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10330	}
10331	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10332	{
10333	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10334	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10335	}
10336
10337	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10338	/* Paranoia. */
10339	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10340	#endif
10341
10342	/*
10343	* No longjmps to ring-3 from this point on!!!
10344	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10345	* This also disables flushing of the R0-logger instance (if any).
10346	*/
10347	VMMRZCallRing3Disable(pVCpu);
10348
10349	/*
10350	* Export the guest state bits.
10351	*
10352	* We cannot perform longjmps while loading the guest state because we do not preserve the
10353	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10354	* CPU migration.
10355	*
10356	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10357	* registers. Hence, loading of the guest state needs to be done -after- injection of events.
10358	*/
10359	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10360	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10361	{ /* likely */ }
10362	else
10363	{
10364	VMMRZCallRing3Enable(pVCpu);
10365	return rcStrict;
10366	}
10367
10368	/*
10369	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10370	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10371	* preemption disabled for a while. Since this is purely to aid the
10372	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10373	* disable interrupt on NT.
10374	*
10375	* We need to check for force-flags that could've possible been altered since we last
10376	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10377	* see @bugref{6398}).
10378	*
10379	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10380	* to ring-3 before executing guest code.
10381	*/
10382	pVmxTransient->fEFlags = ASMIntDisableFlags();
10383
10384	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10385	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10386	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10387	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10388	{
10389	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10390	{
10391	pVCpu->hm.s.Event.fPending = false;
10392
10393	/*
10394	* We've injected any pending events. This is really the point of no return (to ring-3).
10395	*
10396	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10397	* returns from this function, so don't enable them here.
10398	*/
10399	return VINF_SUCCESS;
10400	}
10401
10402	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10403	rcStrict = VINF_EM_RAW_INTERRUPT;
10404	}
10405	else
10406	{
10407	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10408	rcStrict = VINF_EM_RAW_TO_R3;
10409	}
10410
10411	ASMSetFlags(pVmxTransient->fEFlags);
10412	VMMRZCallRing3Enable(pVCpu);
10413
10414	return rcStrict;
10415	}
10416
10417
10418	/**
10419	* Final preparations before executing guest code using hardware-assisted VMX.
10420	*
10421	* We can no longer get preempted to a different host CPU and there are no returns
10422	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10423	* failures), this function is not intended to fail sans unrecoverable hardware
10424	* errors.
10425	*
10426	* @param pVCpu The cross context virtual CPU structure.
10427	* @param pVmxTransient The VMX-transient structure.
10428	*
10429	* @remarks Called with preemption disabled.
10430	* @remarks No-long-jump zone!!!
10431	*/
10432	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10433	{
10434	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10435	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10436	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10437	Assert(!pVCpu->hm.s.Event.fPending);
10438
10439	/*
10440	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10441	*/
10442	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10443	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10444
10445	PVM pVM = pVCpu->CTX_SUFF(pVM);
10446	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10447
10448	if (!CPUMIsGuestFPUStateActive(pVCpu))
10449	{
10450	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10451	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10452	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10453	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10454	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10455	}
10456
10457	/*
10458	* Re-save the host state bits as we may've been preempted (only happens when
10459	* thread-context hooks are used or when the VM start function changes).
10460	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10461	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10462	* see @bugref{8432}.
10463	*
10464	* This may also happen when switching to/from a nested-guest VMCS without leaving
10465	* ring-0.
10466	*/
10467	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10468	{
10469	int rc = hmR0VmxExportHostState(pVCpu);
10470	AssertRC(rc);
10471	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
10472	}
10473	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10474
10475	/*
10476	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10477	*/
10478	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10479	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10480	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10481
10482	/*
10483	* Store status of the shared guest/host debug state at the time of VM-entry.
10484	*/
10485	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
10486	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
10487	{
10488	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
10489	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
10490	}
10491	else
10492	#endif
10493	{
10494	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10495	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10496	}
10497
10498	/*
10499	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10500	* more than one conditional check. The post-run side of our code shall determine
10501	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10502	*/
10503	if (pVmcsInfo->pbVirtApic)
10504	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10505
10506	/*
10507	* Update the host MSRs values in the VM-exit MSR-load area.
10508	*/
10509	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10510	{
10511	if (pVmcsInfo->cExitMsrLoad > 0)
10512	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10513	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10514	}
10515
10516	/*
10517	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10518	* VMX-preemption timer based on the next virtual sync clock deadline.
10519	*/
10520	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10521	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10522	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10523	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10524	{
10525	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10526	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10527	}
10528
10529	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10530	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10531	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10532	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
10533
10534	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10535
10536	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10537	as we're about to start executing the guest . */
10538
10539	/*
10540	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10541	*
10542	* This is done this late as updating the TSC offsetting/preemption timer above
10543	* figures out if we can skip intercepting RDTSCP by calculating the number of
10544	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10545	*/
10546	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10547	{
10548	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10549	{
10550	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10551	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10552	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10553	AssertRC(rc);
10554	}
10555	else
10556	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
10557	}
10558
10559	#ifdef VBOX_STRICT
10560	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo);
10561	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10562	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo));
10563	#endif
10564
10565	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10566	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10567	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs. */
10568	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10569	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10570	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10571	#endif
10572	}
10573
10574
10575	/**
10576	* First C routine invoked after running guest code using hardware-assisted VMX.
10577	*
10578	* @param pVCpu The cross context virtual CPU structure.
10579	* @param pVmxTransient The VMX-transient structure.
10580	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10581	*
10582	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10583	*
10584	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10585	* unconditionally when it is safe to do so.
10586	*/
10587	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10588	{
10589	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10590
10591	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10592	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10593	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10594	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10595	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10596	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10597
10598	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10599	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10600	{
10601	uint64_t uGstTsc;
10602	if (!pVmxTransient->fIsNestedGuest)
10603	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10604	else
10605	{
10606	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10607	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10608	}
10609	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10610	}
10611
10612	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10613	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
10614	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10615
10616	#if HC_ARCH_BITS == 64
10617	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10618	#endif
10619	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
10620	/* The 64-on-32 switcher maintains VMCS-launch state on its own
10621	and we need to leave it alone here. */
10622	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
10623	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10624	#else
10625	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10626	#endif
10627	#ifdef VBOX_STRICT
10628	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10629	#endif
10630	Assert(!ASMIntAreEnabled());
10631	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10632	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10633
10634	/*
10635	* Save the basic VM-exit reason and check if the VM-entry failed.
10636	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10637	*/
10638	uint32_t uExitReason;
10639	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
10640	AssertRC(rc);
10641	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
10642	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
10643
10644	/*
10645	* Check if VMLAUNCH/VMRESUME succeeded.
10646	* If this failed, we cause a guru meditation and cease further execution.
10647	*/
10648	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
10649	{
10650	/*
10651	* Update the VM-exit history array here even if the VM-entry failed due to:
10652	* - Invalid guest state.
10653	* - MSR loading.
10654	* - Machine-check event.
10655	*
10656	* In any of the above cases we will still have a "valid" VM-exit reason
10657	* despite @a fVMEntryFailed being false.
10658	*
10659	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
10660	*
10661	* Note! We don't have CS or RIP at this point. Will probably address that later
10662	* by amending the history entry added here.
10663	*/
10664	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
10665	UINT64_MAX, uHostTsc);
10666
10667	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
10668	{
10669	VMMRZCallRing3Enable(pVCpu);
10670
10671	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10672	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10673
10674	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
10675	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10676	AssertRC(rc);
10677	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
10678	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
10679	AssertRC(rc);
10680	#else
10681	/*
10682	* Import the guest-interruptibility state always as we need it while evaluating
10683	* injecting events on re-entry.
10684	*
10685	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
10686	* checking for real-mode while exporting the state because all bits that cause
10687	* mode changes wrt CR0 are intercepted.
10688	*/
10689	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
10690	AssertRC(rc);
10691	#endif
10692
10693	/*
10694	* Sync the TPR shadow with our APIC state.
10695	*/
10696	if ( !pVmxTransient->fIsNestedGuest
10697	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
10698	{
10699	Assert(pVmcsInfo->pbVirtApic);
10700	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
10701	{
10702	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
10703	AssertRC(rc);
10704	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
10705	}
10706	}
10707
10708	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10709	return;
10710	}
10711	}
10712	else
10713	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
10714
10715	VMMRZCallRing3Enable(pVCpu);
10716	}
10717
10718
10719	/**
10720	* Runs the guest code using hardware-assisted VMX the normal way.
10721	*
10722	* @returns VBox status code.
10723	* @param pVCpu The cross context virtual CPU structure.
10724	* @param pcLoops Pointer to the number of executed loops.
10725	*/
10726	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
10727	{
10728	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10729	Assert(pcLoops);
10730	Assert(*pcLoops <= cMaxResumeLoops);
10731
10732	VMXTRANSIENT VmxTransient;
10733	RT_ZERO(VmxTransient);
10734	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10735
10736	/* Paranoia. */
10737	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
10738	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10739
10740	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10741	for (;;)
10742	{
10743	Assert(!HMR0SuspendPending());
10744	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10745	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10746
10747	/*
10748	* Preparatory work for running nested-guest code, this may force us to
10749	* return to ring-3.
10750	*
10751	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10752	*/
10753	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10754	if (rcStrict != VINF_SUCCESS)
10755	break;
10756
10757	/* Interrupts are disabled at this point! */
10758	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10759	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10760	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10761	/* Interrupts are re-enabled at this point! */
10762
10763	/*
10764	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10765	*/
10766	if (RT_SUCCESS(rcRun))
10767	{ /* very likely */ }
10768	else
10769	{
10770	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10771	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10772	return rcRun;
10773	}
10774
10775	/*
10776	* Profile the VM-exit.
10777	*/
10778	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10779	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10780	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10781	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10782	HMVMX_START_EXIT_DISPATCH_PROF();
10783
10784	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10785
10786	/*
10787	* Handle the VM-exit.
10788	*/
10789	#ifdef HMVMX_USE_FUNCTION_TABLE
10790	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
10791	#else
10792	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
10793	#endif
10794	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10795	if (rcStrict == VINF_SUCCESS)
10796	{
10797	if (++(*pcLoops) <= cMaxResumeLoops)
10798	continue;
10799	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10800	rcStrict = VINF_EM_RAW_INTERRUPT;
10801	}
10802	break;
10803	}
10804
10805	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10806	return rcStrict;
10807	}
10808
10809	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10810	/**
10811	* Runs the nested-guest code using hardware-assisted VMX.
10812	*
10813	* @returns VBox status code.
10814	* @param pVCpu The cross context virtual CPU structure.
10815	* @param pcLoops Pointer to the number of executed loops.
10816	*
10817	* @sa hmR0VmxRunGuestCodeNormal().
10818	*/
10819	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
10820	{
10821	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10822	Assert(pcLoops);
10823	Assert(*pcLoops <= cMaxResumeLoops);
10824
10825	VMXTRANSIENT VmxTransient;
10826	RT_ZERO(VmxTransient);
10827	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10828	VmxTransient.fIsNestedGuest = true;
10829
10830	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10831	for (;;)
10832	{
10833	Assert(!HMR0SuspendPending());
10834	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10835	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10836
10837	/*
10838	* Preparatory work for running guest code, this may force us to
10839	* return to ring-3.
10840	*
10841	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10842	*/
10843	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10844	if (rcStrict != VINF_SUCCESS)
10845	break;
10846
10847	/* Interrupts are disabled at this point! */
10848	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10849	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10850	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10851	/* Interrupts are re-enabled at this point! */
10852
10853	/*
10854	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10855	*/
10856	if (RT_SUCCESS(rcRun))
10857	{ /* very likely */ }
10858	else
10859	{
10860	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10861	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10862	return rcRun;
10863	}
10864
10865	/*
10866	* Profile the VM-exit.
10867	*/
10868	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10869	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10870	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10871	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10872	HMVMX_START_EXIT_DISPATCH_PROF();
10873
10874	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10875
10876	/*
10877	* Handle the VM-exit.
10878	*/
10879	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
10880	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10881	if ( rcStrict == VINF_SUCCESS
10882	&& CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10883	{
10884	if (++(*pcLoops) <= cMaxResumeLoops)
10885	continue;
10886	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10887	rcStrict = VINF_EM_RAW_INTERRUPT;
10888	}
10889	break;
10890	}
10891
10892	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10893	return rcStrict;
10894	}
10895	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10896
10897
10898	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
10899	* probes.
10900	*
10901	* The following few functions and associated structure contains the bloat
10902	* necessary for providing detailed debug events and dtrace probes as well as
10903	* reliable host side single stepping. This works on the principle of
10904	* "subclassing" the normal execution loop and workers. We replace the loop
10905	* method completely and override selected helpers to add necessary adjustments
10906	* to their core operation.
10907	*
10908	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
10909	* any performance for debug and analysis features.
10910	*
10911	* @{
10912	*/
10913
10914	/**
10915	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
10916	* the debug run loop.
10917	*/
10918	typedef struct VMXRUNDBGSTATE
10919	{
10920	/** The RIP we started executing at. This is for detecting that we stepped. */
10921	uint64_t uRipStart;
10922	/** The CS we started executing with. */
10923	uint16_t uCsStart;
10924
10925	/** Whether we've actually modified the 1st execution control field. */
10926	bool fModifiedProcCtls : 1;
10927	/** Whether we've actually modified the 2nd execution control field. */
10928	bool fModifiedProcCtls2 : 1;
10929	/** Whether we've actually modified the exception bitmap. */
10930	bool fModifiedXcptBitmap : 1;
10931
10932	/** We desire the modified the CR0 mask to be cleared. */
10933	bool fClearCr0Mask : 1;
10934	/** We desire the modified the CR4 mask to be cleared. */
10935	bool fClearCr4Mask : 1;
10936	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
10937	uint32_t fCpe1Extra;
10938	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
10939	uint32_t fCpe1Unwanted;
10940	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
10941	uint32_t fCpe2Extra;
10942	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
10943	uint32_t bmXcptExtra;
10944	/** The sequence number of the Dtrace provider settings the state was
10945	* configured against. */
10946	uint32_t uDtraceSettingsSeqNo;
10947	/** VM-exits to check (one bit per VM-exit). */
10948	uint32_t bmExitsToCheck[3];
10949
10950	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
10951	uint32_t fProcCtlsInitial;
10952	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
10953	uint32_t fProcCtls2Initial;
10954	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
10955	uint32_t bmXcptInitial;
10956	} VMXRUNDBGSTATE;
10957	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
10958	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
10959
10960
10961	/**
10962	* Initializes the VMXRUNDBGSTATE structure.
10963	*
10964	* @param pVCpu The cross context virtual CPU structure of the
10965	* calling EMT.
10966	* @param pVmxTransient The VMX-transient structure.
10967	* @param pDbgState The debug state to initialize.
10968	*/
10969	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
10970	{
10971	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
10972	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
10973
10974	pDbgState->fModifiedProcCtls = false;
10975	pDbgState->fModifiedProcCtls2 = false;
10976	pDbgState->fModifiedXcptBitmap = false;
10977	pDbgState->fClearCr0Mask = false;
10978	pDbgState->fClearCr4Mask = false;
10979	pDbgState->fCpe1Extra = 0;
10980	pDbgState->fCpe1Unwanted = 0;
10981	pDbgState->fCpe2Extra = 0;
10982	pDbgState->bmXcptExtra = 0;
10983	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
10984	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
10985	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
10986	}
10987
10988
10989	/**
10990	* Updates the VMSC fields with changes requested by @a pDbgState.
10991	*
10992	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
10993	* immediately before executing guest code, i.e. when interrupts are disabled.
10994	* We don't check status codes here as we cannot easily assert or return in the
10995	* latter case.
10996	*
10997	* @param pVCpu The cross context virtual CPU structure.
10998	* @param pVmxTransient The VMX-transient structure.
10999	* @param pDbgState The debug state.
11000	*/
11001	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11002	{
11003	/*
11004	* Ensure desired flags in VMCS control fields are set.
11005	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11006	*
11007	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11008	* there should be no stale data in pCtx at this point.
11009	*/
11010	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11011	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11012	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11013	{
11014	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11015	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11016	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11017	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11018	pDbgState->fModifiedProcCtls = true;
11019	}
11020
11021	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11022	{
11023	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11024	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11025	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11026	pDbgState->fModifiedProcCtls2 = true;
11027	}
11028
11029	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11030	{
11031	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11032	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11033	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11034	pDbgState->fModifiedXcptBitmap = true;
11035	}
11036
11037	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11038	{
11039	pVmcsInfo->u64Cr0Mask = 0;
11040	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
11041	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11042	}
11043
11044	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11045	{
11046	pVmcsInfo->u64Cr4Mask = 0;
11047	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
11048	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11049	}
11050
11051	NOREF(pVCpu);
11052	}
11053
11054
11055	/**
11056	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11057	* re-entry next time around.
11058	*
11059	* @returns Strict VBox status code (i.e. informational status codes too).
11060	* @param pVCpu The cross context virtual CPU structure.
11061	* @param pVmxTransient The VMX-transient structure.
11062	* @param pDbgState The debug state.
11063	* @param rcStrict The return code from executing the guest using single
11064	* stepping.
11065	*/
11066	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11067	VBOXSTRICTRC rcStrict)
11068	{
11069	/*
11070	* Restore VM-exit control settings as we may not reenter this function the
11071	* next time around.
11072	*/
11073	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11074
11075	/* We reload the initial value, trigger what we can of recalculations the
11076	next time around. From the looks of things, that's all that's required atm. */
11077	if (pDbgState->fModifiedProcCtls)
11078	{
11079	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11080	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11081	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11082	AssertRCReturn(rc2, rc2);
11083	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11084	}
11085
11086	/* We're currently the only ones messing with this one, so just restore the
11087	cached value and reload the field. */
11088	if ( pDbgState->fModifiedProcCtls2
11089	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11090	{
11091	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11092	AssertRCReturn(rc2, rc2);
11093	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11094	}
11095
11096	/* If we've modified the exception bitmap, we restore it and trigger
11097	reloading and partial recalculation the next time around. */
11098	if (pDbgState->fModifiedXcptBitmap)
11099	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11100
11101	return rcStrict;
11102	}
11103
11104
11105	/**
11106	* Configures VM-exit controls for current DBGF and DTrace settings.
11107	*
11108	* This updates @a pDbgState and the VMCS execution control fields to reflect
11109	* the necessary VM-exits demanded by DBGF and DTrace.
11110	*
11111	* @param pVCpu The cross context virtual CPU structure.
11112	* @param pVmxTransient The VMX-transient structure. May update
11113	* fUpdatedTscOffsettingAndPreemptTimer.
11114	* @param pDbgState The debug state.
11115	*/
11116	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11117	{
11118	/*
11119	* Take down the dtrace serial number so we can spot changes.
11120	*/
11121	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11122	ASMCompilerBarrier();
11123
11124	/*
11125	* We'll rebuild most of the middle block of data members (holding the
11126	* current settings) as we go along here, so start by clearing it all.
11127	*/
11128	pDbgState->bmXcptExtra = 0;
11129	pDbgState->fCpe1Extra = 0;
11130	pDbgState->fCpe1Unwanted = 0;
11131	pDbgState->fCpe2Extra = 0;
11132	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11133	pDbgState->bmExitsToCheck[i] = 0;
11134
11135	/*
11136	* Software interrupts (INT XXh) - no idea how to trigger these...
11137	*/
11138	PVM pVM = pVCpu->CTX_SUFF(pVM);
11139	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11140	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11141	{
11142	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11143	}
11144
11145	/*
11146	* INT3 breakpoints - triggered by #BP exceptions.
11147	*/
11148	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11149	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11150
11151	/*
11152	* Exception bitmap and XCPT events+probes.
11153	*/
11154	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11155	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11156	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11157
11158	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11159	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11160	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11161	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11162	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11163	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11164	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11165	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11166	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11167	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11168	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11169	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11170	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11171	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11172	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11173	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11174	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11175	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11176
11177	if (pDbgState->bmXcptExtra)
11178	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11179
11180	/*
11181	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11182	*
11183	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11184	* So, when adding/changing/removing please don't forget to update it.
11185	*
11186	* Some of the macros are picking up local variables to save horizontal space,
11187	* (being able to see it in a table is the lesser evil here).
11188	*/
11189	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11190	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11191	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11192	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11193	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11194	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11195	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11196	} else do { } while (0)
11197	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11198	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11199	{ \
11200	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11201	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11202	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11203	} else do { } while (0)
11204	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11205	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11206	{ \
11207	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11208	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11209	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11210	} else do { } while (0)
11211	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11212	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11213	{ \
11214	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11215	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11216	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11217	} else do { } while (0)
11218
11219	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11220	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11221	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11222	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11223	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11224
11225	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11226	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11227	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11228	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11229	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11230	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11231	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11232	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11233	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11234	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11235	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11236	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11237	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11238	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11239	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11240	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11241	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11242	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11243	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11244	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11245	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11246	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11247	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11248	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11249	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11250	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11251	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11252	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11253	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11254	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11255	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11256	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11257	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11258	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11259	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11260	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11261
11262	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11263	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11264	{
11265	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11266	\| CPUMCTX_EXTRN_APIC_TPR);
11267	AssertRC(rc);
11268
11269	#if 0 /** @todo fix me */
11270	pDbgState->fClearCr0Mask = true;
11271	pDbgState->fClearCr4Mask = true;
11272	#endif
11273	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11274	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11275	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11276	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11277	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11278	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11279	require clearing here and in the loop if we start using it. */
11280	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11281	}
11282	else
11283	{
11284	if (pDbgState->fClearCr0Mask)
11285	{
11286	pDbgState->fClearCr0Mask = false;
11287	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11288	}
11289	if (pDbgState->fClearCr4Mask)
11290	{
11291	pDbgState->fClearCr4Mask = false;
11292	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11293	}
11294	}
11295	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11296	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11297
11298	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11299	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11300	{
11301	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11302	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11303	}
11304	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11305	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11306
11307	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11308	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11309	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11310	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11311	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11312	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11313	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11314	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11315	#if 0 /** @todo too slow, fix handler. */
11316	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11317	#endif
11318	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11319
11320	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11321	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11322	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11323	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11324	{
11325	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11326	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11327	}
11328	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11329	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11330	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11331	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11332
11333	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11334	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11335	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11336	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11337	{
11338	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11339	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11340	}
11341	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11342	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11343	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11344	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11345
11346	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11347	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11348	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11349	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11350	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11351	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11352	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11353	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11354	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11355	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11356	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11357	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11358	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11359	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11360	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11361	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11362	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11363	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11364	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11365	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11366	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11367	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11368
11369	#undef IS_EITHER_ENABLED
11370	#undef SET_ONLY_XBM_IF_EITHER_EN
11371	#undef SET_CPE1_XBM_IF_EITHER_EN
11372	#undef SET_CPEU_XBM_IF_EITHER_EN
11373	#undef SET_CPE2_XBM_IF_EITHER_EN
11374
11375	/*
11376	* Sanitize the control stuff.
11377	*/
11378	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11379	if (pDbgState->fCpe2Extra)
11380	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11381	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11382	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11383	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11384	{
11385	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11386	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11387	}
11388
11389	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11390	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11391	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11392	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11393	}
11394
11395
11396	/**
11397	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11398	* appropriate.
11399	*
11400	* The caller has checked the VM-exit against the
11401	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11402	* already, so we don't have to do that either.
11403	*
11404	* @returns Strict VBox status code (i.e. informational status codes too).
11405	* @param pVCpu The cross context virtual CPU structure.
11406	* @param pVmxTransient The VMX-transient structure.
11407	* @param uExitReason The VM-exit reason.
11408	*
11409	* @remarks The name of this function is displayed by dtrace, so keep it short
11410	* and to the point. No longer than 33 chars long, please.
11411	*/
11412	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11413	{
11414	/*
11415	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11416	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11417	*
11418	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11419	* does. Must add/change/remove both places. Same ordering, please.
11420	*
11421	* Added/removed events must also be reflected in the next section
11422	* where we dispatch dtrace events.
11423	*/
11424	bool fDtrace1 = false;
11425	bool fDtrace2 = false;
11426	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11427	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11428	uint32_t uEventArg = 0;
11429	#define SET_EXIT(a_EventSubName) \
11430	do { \
11431	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11432	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11433	} while (0)
11434	#define SET_BOTH(a_EventSubName) \
11435	do { \
11436	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11437	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11438	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11439	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11440	} while (0)
11441	switch (uExitReason)
11442	{
11443	case VMX_EXIT_MTF:
11444	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11445
11446	case VMX_EXIT_XCPT_OR_NMI:
11447	{
11448	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11449	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11450	{
11451	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11452	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11453	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11454	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11455	{
11456	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11457	{
11458	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11459	uEventArg = pVmxTransient->uExitIntErrorCode;
11460	}
11461	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11462	switch (enmEvent1)
11463	{
11464	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11465	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11466	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11467	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11468	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11469	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11470	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11471	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11472	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11473	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11474	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11475	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11476	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11477	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11478	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11479	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11480	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11481	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11482	default: break;
11483	}
11484	}
11485	else
11486	AssertFailed();
11487	break;
11488
11489	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11490	uEventArg = idxVector;
11491	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11492	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11493	break;
11494	}
11495	break;
11496	}
11497
11498	case VMX_EXIT_TRIPLE_FAULT:
11499	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11500	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11501	break;
11502	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11503	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11504	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11505	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11506	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11507
11508	/* Instruction specific VM-exits: */
11509	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11510	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11511	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11512	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11513	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11514	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11515	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11516	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11517	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11518	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11519	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11520	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11521	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11522	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11523	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11524	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11525	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11526	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11527	case VMX_EXIT_MOV_CRX:
11528	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11529	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11530	SET_BOTH(CRX_READ);
11531	else
11532	SET_BOTH(CRX_WRITE);
11533	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11534	break;
11535	case VMX_EXIT_MOV_DRX:
11536	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11537	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11538	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11539	SET_BOTH(DRX_READ);
11540	else
11541	SET_BOTH(DRX_WRITE);
11542	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11543	break;
11544	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11545	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11546	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11547	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11548	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11549	case VMX_EXIT_GDTR_IDTR_ACCESS:
11550	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11551	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11552	{
11553	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11554	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11555	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11556	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11557	}
11558	break;
11559
11560	case VMX_EXIT_LDTR_TR_ACCESS:
11561	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11562	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11563	{
11564	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11565	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11566	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11567	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11568	}
11569	break;
11570
11571	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11572	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11573	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11574	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11575	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11576	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11577	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11578	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11579	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11580	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11581	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11582
11583	/* Events that aren't relevant at this point. */
11584	case VMX_EXIT_EXT_INT:
11585	case VMX_EXIT_INT_WINDOW:
11586	case VMX_EXIT_NMI_WINDOW:
11587	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11588	case VMX_EXIT_PREEMPT_TIMER:
11589	case VMX_EXIT_IO_INSTR:
11590	break;
11591
11592	/* Errors and unexpected events. */
11593	case VMX_EXIT_INIT_SIGNAL:
11594	case VMX_EXIT_SIPI:
11595	case VMX_EXIT_IO_SMI:
11596	case VMX_EXIT_SMI:
11597	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11598	case VMX_EXIT_ERR_MSR_LOAD:
11599	case VMX_EXIT_ERR_MACHINE_CHECK:
11600	break;
11601
11602	default:
11603	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11604	break;
11605	}
11606	#undef SET_BOTH
11607	#undef SET_EXIT
11608
11609	/*
11610	* Dtrace tracepoints go first. We do them here at once so we don't
11611	* have to copy the guest state saving and stuff a few dozen times.
11612	* Down side is that we've got to repeat the switch, though this time
11613	* we use enmEvent since the probes are a subset of what DBGF does.
11614	*/
11615	if (fDtrace1 \|\| fDtrace2)
11616	{
11617	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11618	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11619	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11620	switch (enmEvent1)
11621	{
11622	/** @todo consider which extra parameters would be helpful for each probe. */
11623	case DBGFEVENT_END: break;
11624	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11625	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11626	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11627	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11628	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11629	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11630	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11631	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11632	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
11633	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
11634	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
11635	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
11636	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
11637	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
11638	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
11639	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
11640	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
11641	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
11642	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11643	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11644	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
11645	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
11646	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
11647	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
11648	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
11649	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
11650	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
11651	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11652	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11653	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11654	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11655	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11656	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
11657	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11658	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
11659	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
11660	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
11661	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
11662	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
11663	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
11664	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
11665	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
11666	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
11667	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
11668	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
11669	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
11670	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
11671	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
11672	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
11673	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
11674	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
11675	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
11676	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
11677	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
11678	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
11679	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
11680	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
11681	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
11682	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
11683	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
11684	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
11685	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
11686	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
11687	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
11688	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
11689	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
11690	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
11691	}
11692	switch (enmEvent2)
11693	{
11694	/** @todo consider which extra parameters would be helpful for each probe. */
11695	case DBGFEVENT_END: break;
11696	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
11697	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11698	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
11699	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
11700	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
11701	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
11702	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
11703	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
11704	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
11705	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11706	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11707	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11708	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11709	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11710	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
11711	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11712	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
11713	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
11714	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
11715	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
11716	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
11717	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
11718	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
11719	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
11720	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
11721	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
11722	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
11723	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
11724	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
11725	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
11726	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
11727	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
11728	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
11729	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
11730	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
11731	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
11732	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
11733	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
11734	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
11735	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
11736	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
11737	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
11738	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
11739	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
11740	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
11741	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
11742	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
11743	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
11744	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
11745	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
11746	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
11747	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
11748	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
11749	}
11750	}
11751
11752	/*
11753	* Fire of the DBGF event, if enabled (our check here is just a quick one,
11754	* the DBGF call will do a full check).
11755	*
11756	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
11757	* Note! If we have to events, we prioritize the first, i.e. the instruction
11758	* one, in order to avoid event nesting.
11759	*/
11760	PVM pVM = pVCpu->CTX_SUFF(pVM);
11761	if ( enmEvent1 != DBGFEVENT_END
11762	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
11763	{
11764	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11765	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
11766	if (rcStrict != VINF_SUCCESS)
11767	return rcStrict;
11768	}
11769	else if ( enmEvent2 != DBGFEVENT_END
11770	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
11771	{
11772	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11773	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
11774	if (rcStrict != VINF_SUCCESS)
11775	return rcStrict;
11776	}
11777
11778	return VINF_SUCCESS;
11779	}
11780
11781
11782	/**
11783	* Single-stepping VM-exit filtering.
11784	*
11785	* This is preprocessing the VM-exits and deciding whether we've gotten far
11786	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
11787	* handling is performed.
11788	*
11789	* @returns Strict VBox status code (i.e. informational status codes too).
11790	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
11791	* @param pVmxTransient The VMX-transient structure.
11792	* @param pDbgState The debug state.
11793	*/
11794	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11795	{
11796	/*
11797	* Expensive (saves context) generic dtrace VM-exit probe.
11798	*/
11799	uint32_t const uExitReason = pVmxTransient->uExitReason;
11800	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
11801	{ /* more likely */ }
11802	else
11803	{
11804	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11805	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11806	AssertRC(rc);
11807	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
11808	}
11809
11810	/*
11811	* Check for host NMI, just to get that out of the way.
11812	*/
11813	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
11814	{ /* normally likely */ }
11815	else
11816	{
11817	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11818	AssertRCReturn(rc2, rc2);
11819	uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11820	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11821	return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
11822	}
11823
11824	/*
11825	* Check for single stepping event if we're stepping.
11826	*/
11827	if (pVCpu->hm.s.fSingleInstruction)
11828	{
11829	switch (uExitReason)
11830	{
11831	case VMX_EXIT_MTF:
11832	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11833
11834	/* Various events: */
11835	case VMX_EXIT_XCPT_OR_NMI:
11836	case VMX_EXIT_EXT_INT:
11837	case VMX_EXIT_TRIPLE_FAULT:
11838	case VMX_EXIT_INT_WINDOW:
11839	case VMX_EXIT_NMI_WINDOW:
11840	case VMX_EXIT_TASK_SWITCH:
11841	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11842	case VMX_EXIT_APIC_ACCESS:
11843	case VMX_EXIT_EPT_VIOLATION:
11844	case VMX_EXIT_EPT_MISCONFIG:
11845	case VMX_EXIT_PREEMPT_TIMER:
11846
11847	/* Instruction specific VM-exits: */
11848	case VMX_EXIT_CPUID:
11849	case VMX_EXIT_GETSEC:
11850	case VMX_EXIT_HLT:
11851	case VMX_EXIT_INVD:
11852	case VMX_EXIT_INVLPG:
11853	case VMX_EXIT_RDPMC:
11854	case VMX_EXIT_RDTSC:
11855	case VMX_EXIT_RSM:
11856	case VMX_EXIT_VMCALL:
11857	case VMX_EXIT_VMCLEAR:
11858	case VMX_EXIT_VMLAUNCH:
11859	case VMX_EXIT_VMPTRLD:
11860	case VMX_EXIT_VMPTRST:
11861	case VMX_EXIT_VMREAD:
11862	case VMX_EXIT_VMRESUME:
11863	case VMX_EXIT_VMWRITE:
11864	case VMX_EXIT_VMXOFF:
11865	case VMX_EXIT_VMXON:
11866	case VMX_EXIT_MOV_CRX:
11867	case VMX_EXIT_MOV_DRX:
11868	case VMX_EXIT_IO_INSTR:
11869	case VMX_EXIT_RDMSR:
11870	case VMX_EXIT_WRMSR:
11871	case VMX_EXIT_MWAIT:
11872	case VMX_EXIT_MONITOR:
11873	case VMX_EXIT_PAUSE:
11874	case VMX_EXIT_GDTR_IDTR_ACCESS:
11875	case VMX_EXIT_LDTR_TR_ACCESS:
11876	case VMX_EXIT_INVEPT:
11877	case VMX_EXIT_RDTSCP:
11878	case VMX_EXIT_INVVPID:
11879	case VMX_EXIT_WBINVD:
11880	case VMX_EXIT_XSETBV:
11881	case VMX_EXIT_RDRAND:
11882	case VMX_EXIT_INVPCID:
11883	case VMX_EXIT_VMFUNC:
11884	case VMX_EXIT_RDSEED:
11885	case VMX_EXIT_XSAVES:
11886	case VMX_EXIT_XRSTORS:
11887	{
11888	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11889	AssertRCReturn(rc, rc);
11890	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
11891	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
11892	return VINF_EM_DBG_STEPPED;
11893	break;
11894	}
11895
11896	/* Errors and unexpected events: */
11897	case VMX_EXIT_INIT_SIGNAL:
11898	case VMX_EXIT_SIPI:
11899	case VMX_EXIT_IO_SMI:
11900	case VMX_EXIT_SMI:
11901	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11902	case VMX_EXIT_ERR_MSR_LOAD:
11903	case VMX_EXIT_ERR_MACHINE_CHECK:
11904	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
11905	break;
11906
11907	default:
11908	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11909	break;
11910	}
11911	}
11912
11913	/*
11914	* Check for debugger event breakpoints and dtrace probes.
11915	*/
11916	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
11917	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
11918	{
11919	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
11920	if (rcStrict != VINF_SUCCESS)
11921	return rcStrict;
11922	}
11923
11924	/*
11925	* Normal processing.
11926	*/
11927	#ifdef HMVMX_USE_FUNCTION_TABLE
11928	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
11929	#else
11930	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
11931	#endif
11932	}
11933
11934
11935	/**
11936	* Single steps guest code using hardware-assisted VMX.
11937	*
11938	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
11939	* but single-stepping through the hypervisor debugger.
11940	*
11941	* @returns Strict VBox status code (i.e. informational status codes too).
11942	* @param pVCpu The cross context virtual CPU structure.
11943	* @param pcLoops Pointer to the number of executed loops.
11944	*
11945	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
11946	*/
11947	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
11948	{
11949	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11950	Assert(pcLoops);
11951	Assert(*pcLoops <= cMaxResumeLoops);
11952
11953	VMXTRANSIENT VmxTransient;
11954	RT_ZERO(VmxTransient);
11955	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11956
11957	/* Set HMCPU indicators. */
11958	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
11959	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
11960	pVCpu->hm.s.fDebugWantRdTscExit = false;
11961	pVCpu->hm.s.fUsingDebugLoop = true;
11962
11963	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
11964	VMXRUNDBGSTATE DbgState;
11965	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
11966	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
11967
11968	/*
11969	* The loop.
11970	*/
11971	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11972	for (;;)
11973	{
11974	Assert(!HMR0SuspendPending());
11975	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11976	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11977	bool fStepping = pVCpu->hm.s.fSingleInstruction;
11978
11979	/* Set up VM-execution controls the next two can respond to. */
11980	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
11981
11982	/*
11983	* Preparatory work for running guest code, this may force us to
11984	* return to ring-3.
11985	*
11986	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11987	*/
11988	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
11989	if (rcStrict != VINF_SUCCESS)
11990	break;
11991
11992	/* Interrupts are disabled at this point! */
11993	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11994
11995	/* Override any obnoxious code in the above two calls. */
11996	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
11997
11998	/*
11999	* Finally execute the guest.
12000	*/
12001	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12002
12003	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12004	/* Interrupts are re-enabled at this point! */
12005
12006	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12007	if (RT_SUCCESS(rcRun))
12008	{ /* very likely */ }
12009	else
12010	{
12011	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12012	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12013	return rcRun;
12014	}
12015
12016	/* Profile the VM-exit. */
12017	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12018	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12019	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12020	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12021	HMVMX_START_EXIT_DISPATCH_PROF();
12022
12023	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12024
12025	/*
12026	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12027	*/
12028	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12029	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12030	if (rcStrict != VINF_SUCCESS)
12031	break;
12032	if (++(*pcLoops) > cMaxResumeLoops)
12033	{
12034	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12035	rcStrict = VINF_EM_RAW_INTERRUPT;
12036	break;
12037	}
12038
12039	/*
12040	* Stepping: Did the RIP change, if so, consider it a single step.
12041	* Otherwise, make sure one of the TFs gets set.
12042	*/
12043	if (fStepping)
12044	{
12045	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12046	AssertRC(rc);
12047	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12048	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12049	{
12050	rcStrict = VINF_EM_DBG_STEPPED;
12051	break;
12052	}
12053	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12054	}
12055
12056	/*
12057	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12058	*/
12059	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12060	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12061	}
12062
12063	/*
12064	* Clear the X86_EFL_TF if necessary.
12065	*/
12066	if (pVCpu->hm.s.fClearTrapFlag)
12067	{
12068	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12069	AssertRC(rc);
12070	pVCpu->hm.s.fClearTrapFlag = false;
12071	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12072	}
12073	/** @todo there seems to be issues with the resume flag when the monitor trap
12074	* flag is pending without being used. Seen early in bios init when
12075	* accessing APIC page in protected mode. */
12076
12077	/*
12078	* Restore VM-exit control settings as we may not re-enter this function the
12079	* next time around.
12080	*/
12081	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12082
12083	/* Restore HMCPU indicators. */
12084	pVCpu->hm.s.fUsingDebugLoop = false;
12085	pVCpu->hm.s.fDebugWantRdTscExit = false;
12086	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12087
12088	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12089	return rcStrict;
12090	}
12091
12092
12093	/** @} */
12094
12095
12096	/**
12097	* Checks if any expensive dtrace probes are enabled and we should go to the
12098	* debug loop.
12099	*
12100	* @returns true if we should use debug loop, false if not.
12101	*/
12102	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12103	{
12104	/* It's probably faster to OR the raw 32-bit counter variables together.
12105	Since the variables are in an array and the probes are next to one
12106	another (more or less), we have good locality. So, better read
12107	eight-nine cache lines ever time and only have one conditional, than
12108	128+ conditionals, right? */
12109	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12110	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12111	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12112	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12113	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12114	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12115	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12116	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12117	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12118	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12119	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12120	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12121	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12122	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12123	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12124	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12125	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12126	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12127	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12128	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12129	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12130	) != 0
12131	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12132	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12133	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12134	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12135	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12136	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12137	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12138	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12139	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12140	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12141	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12142	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12143	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12144	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12145	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12146	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12147	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12148	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12149	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12150	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12151	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12152	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12153	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12154	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12155	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12156	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12157	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12158	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12159	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12160	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12161	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12162	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12163	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12164	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12165	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12166	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12167	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12168	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12169	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12170	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12171	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12172	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12173	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12174	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12175	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12176	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12177	) != 0
12178	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12179	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12180	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12181	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12182	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12183	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12184	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12185	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12186	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12187	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12188	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12189	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12190	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12191	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12192	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12193	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12194	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12195	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12196	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12197	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12198	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12199	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12200	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12201	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12202	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12203	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12204	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12205	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12206	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12207	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12208	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12209	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12210	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12211	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12212	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12213	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12214	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12215	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12216	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12217	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12218	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12219	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12220	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12221	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12222	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12223	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12224	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12225	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12226	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12227	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12228	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12229	) != 0;
12230	}
12231
12232
12233	/**
12234	* Runs the guest using hardware-assisted VMX.
12235	*
12236	* @returns Strict VBox status code (i.e. informational status codes too).
12237	* @param pVCpu The cross context virtual CPU structure.
12238	*/
12239	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
12240	{
12241	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12242	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12243	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12244	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12245
12246	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
12247
12248	VBOXSTRICTRC rcStrict;
12249	uint32_t cLoops = 0;
12250	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12251	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12252	#else
12253	bool const fInNestedGuestMode = false;
12254	#endif
12255	if (!fInNestedGuestMode)
12256	{
12257	if ( !pVCpu->hm.s.fUseDebugLoop
12258	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12259	&& !DBGFIsStepping(pVCpu)
12260	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12261	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12262	else
12263	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12264	}
12265	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12266	else
12267	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
12268
12269	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12270	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12271	#endif
12272
12273	if (rcStrict == VERR_EM_INTERPRETER)
12274	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12275	else if (rcStrict == VINF_EM_RESET)
12276	rcStrict = VINF_EM_TRIPLE_FAULT;
12277
12278	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12279	if (RT_FAILURE(rc2))
12280	{
12281	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12282	rcStrict = rc2;
12283	}
12284	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12285	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12286	return rcStrict;
12287	}
12288
12289
12290	#ifndef HMVMX_USE_FUNCTION_TABLE
12291	/**
12292	* Handles a guest VM-exit from hardware-assisted VMX execution.
12293	*
12294	* @returns Strict VBox status code (i.e. informational status codes too).
12295	* @param pVCpu The cross context virtual CPU structure.
12296	* @param pVmxTransient The VMX-transient structure.
12297	*/
12298	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12299	{
12300	#ifdef DEBUG_ramshankar
12301	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12302	do { \
12303	if (a_fSave != 0) \
12304	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
12305	VBOXSTRICTRC rcStrict = a_CallExpr; \
12306	if (a_fSave != 0) \
12307	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12308	return rcStrict; \
12309	} while (0)
12310	#else
12311	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12312	#endif
12313	uint32_t const rcReason = pVmxTransient->uExitReason;
12314	switch (rcReason)
12315	{
12316	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12317	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12318	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12319	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12320	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12321	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12322	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12323	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12324	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12325	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12326	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12327	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12328	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12329	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12330	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12331	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12332	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12333	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12334	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12335	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12336	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12337	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12338	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12339	case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
12340	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12341	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12342	case VMX_EXIT_GDTR_IDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12343	case VMX_EXIT_LDTR_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12344	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12345	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12346	case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
12347	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12348	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12349	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12350	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12351	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12352	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12353	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12354	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12355	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12356	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12357	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12358	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12359	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12360	#else
12361	case VMX_EXIT_VMCLEAR:
12362	case VMX_EXIT_VMLAUNCH:
12363	case VMX_EXIT_VMPTRLD:
12364	case VMX_EXIT_VMPTRST:
12365	case VMX_EXIT_VMREAD:
12366	case VMX_EXIT_VMRESUME:
12367	case VMX_EXIT_VMWRITE:
12368	case VMX_EXIT_VMXOFF:
12369	case VMX_EXIT_VMXON:
12370	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12371	#endif
12372
12373	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12374	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12375	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
12376	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
12377	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
12378	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
12379	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
12380	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12381	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
12382
12383	case VMX_EXIT_INVEPT:
12384	case VMX_EXIT_INVVPID:
12385	case VMX_EXIT_VMFUNC:
12386	case VMX_EXIT_XSAVES:
12387	case VMX_EXIT_XRSTORS:
12388	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12389
12390	case VMX_EXIT_ENCLS:
12391	case VMX_EXIT_RDSEED:
12392	case VMX_EXIT_PML_FULL:
12393	default:
12394	return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
12395	}
12396	#undef VMEXIT_CALL_RET
12397	}
12398	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12399
12400
12401	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12402	/**
12403	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12404	*
12405	* @returns Strict VBox status code (i.e. informational status codes too).
12406	* @param pVCpu The cross context virtual CPU structure.
12407	* @param pVmxTransient The VMX-transient structure.
12408	*/
12409	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12410	{
12411	uint32_t const rcReason = pVmxTransient->uExitReason;
12412	switch (rcReason)
12413	{
12414	case VMX_EXIT_EPT_MISCONFIG:
12415	case VMX_EXIT_EPT_VIOLATION:
12416	case VMX_EXIT_IO_INSTR:
12417	case VMX_EXIT_CPUID:
12418	case VMX_EXIT_RDTSC:
12419	case VMX_EXIT_RDTSCP:
12420	case VMX_EXIT_APIC_ACCESS:
12421	case VMX_EXIT_XCPT_OR_NMI:
12422	case VMX_EXIT_MOV_CRX:
12423	case VMX_EXIT_EXT_INT:
12424	case VMX_EXIT_INT_WINDOW:
12425	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12426	case VMX_EXIT_MWAIT:
12427	case VMX_EXIT_MONITOR:
12428	case VMX_EXIT_TASK_SWITCH:
12429	case VMX_EXIT_PREEMPT_TIMER:
12430	case VMX_EXIT_RDMSR:
12431	case VMX_EXIT_WRMSR:
12432	case VMX_EXIT_VMCALL:
12433	case VMX_EXIT_MOV_DRX:
12434	case VMX_EXIT_HLT:
12435	case VMX_EXIT_INVD:
12436	case VMX_EXIT_INVLPG:
12437	case VMX_EXIT_RSM:
12438	case VMX_EXIT_MTF:
12439	case VMX_EXIT_PAUSE:
12440	case VMX_EXIT_GDTR_IDTR_ACCESS:
12441	case VMX_EXIT_LDTR_TR_ACCESS:
12442	case VMX_EXIT_WBINVD:
12443	case VMX_EXIT_XSETBV:
12444	case VMX_EXIT_RDRAND:
12445	case VMX_EXIT_INVPCID:
12446	case VMX_EXIT_GETSEC:
12447	case VMX_EXIT_RDPMC:
12448	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12449	case VMX_EXIT_VMCLEAR:
12450	case VMX_EXIT_VMLAUNCH:
12451	case VMX_EXIT_VMPTRLD:
12452	case VMX_EXIT_VMPTRST:
12453	case VMX_EXIT_VMREAD:
12454	case VMX_EXIT_VMRESUME:
12455	case VMX_EXIT_VMWRITE:
12456	case VMX_EXIT_VMXOFF:
12457	case VMX_EXIT_VMXON:
12458	#endif
12459	case VMX_EXIT_TRIPLE_FAULT:
12460	case VMX_EXIT_NMI_WINDOW:
12461	case VMX_EXIT_INIT_SIGNAL:
12462	case VMX_EXIT_SIPI:
12463	case VMX_EXIT_IO_SMI:
12464	case VMX_EXIT_SMI:
12465	case VMX_EXIT_ERR_MSR_LOAD:
12466	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12467	case VMX_EXIT_ERR_MACHINE_CHECK:
12468
12469	case VMX_EXIT_INVEPT:
12470	case VMX_EXIT_INVVPID:
12471	case VMX_EXIT_VMFUNC:
12472	case VMX_EXIT_XSAVES:
12473	case VMX_EXIT_XRSTORS:
12474
12475	case VMX_EXIT_ENCLS:
12476	case VMX_EXIT_RDSEED:
12477	case VMX_EXIT_PML_FULL:
12478	default:
12479	return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
12480	}
12481	#undef VMEXIT_CALL_RET
12482	}
12483	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12484
12485
12486	#ifdef VBOX_STRICT
12487	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
12488	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
12489	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
12490
12491	# define HMVMX_ASSERT_PREEMPT_CPUID() \
12492	do { \
12493	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
12494	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
12495	} while (0)
12496
12497	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12498	do { \
12499	AssertPtr((a_pVCpu)); \
12500	AssertPtr((a_pVmxTransient)); \
12501	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
12502	Assert((a_pVmxTransient)->pVmcsInfo); \
12503	Assert(ASMIntAreEnabled()); \
12504	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12505	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
12506	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
12507	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12508	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
12509	HMVMX_ASSERT_PREEMPT_CPUID(); \
12510	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12511	} while (0)
12512
12513	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12514	do { \
12515	Log4Func(("\n")); \
12516	} while (0)
12517	#else
12518	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12519	do { \
12520	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12521	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
12522	} while (0)
12523	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
12524	#endif
12525
12526
12527	/**
12528	* Advances the guest RIP by the specified number of bytes.
12529	*
12530	* @param pVCpu The cross context virtual CPU structure.
12531	* @param cbInstr Number of bytes to advance the RIP by.
12532	*
12533	* @remarks No-long-jump zone!!!
12534	*/
12535	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
12536	{
12537	/* Advance the RIP. */
12538	pVCpu->cpum.GstCtx.rip += cbInstr;
12539	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12540
12541	/* Update interrupt inhibition. */
12542	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
12543	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
12544	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12545	}
12546
12547
12548	/**
12549	* Advances the guest RIP after reading it from the VMCS.
12550	*
12551	* @returns VBox status code, no informational status codes.
12552	* @param pVCpu The cross context virtual CPU structure.
12553	* @param pVmxTransient The VMX-transient structure.
12554	*
12555	* @remarks No-long-jump zone!!!
12556	*/
12557	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12558	{
12559	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12560	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
12561	AssertRCReturn(rc, rc);
12562
12563	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
12564	return VINF_SUCCESS;
12565	}
12566
12567
12568	/**
12569	* Handle a condition that occurred while delivering an event through the guest
12570	* IDT.
12571	*
12572	* @returns Strict VBox status code (i.e. informational status codes too).
12573	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
12574	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
12575	* to continue execution of the guest which will delivery the \#DF.
12576	* @retval VINF_EM_RESET if we detected a triple-fault condition.
12577	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
12578	*
12579	* @param pVCpu The cross context virtual CPU structure.
12580	* @param pVmxTransient The VMX-transient structure.
12581	*
12582	* @remarks No-long-jump zone!!!
12583	*/
12584	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12585	{
12586	uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12587
12588	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12589	rc2 \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12590	AssertRCReturn(rc2, rc2);
12591
12592	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
12593	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12594	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12595	{
12596	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12597	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12598
12599	/*
12600	* If the event was a software interrupt (generated with INT n) or a software exception
12601	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
12602	* can handle the VM-exit and continue guest execution which will re-execute the
12603	* instruction rather than re-injecting the exception, as that can cause premature
12604	* trips to ring-3 before injection and involve TRPM which currently has no way of
12605	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
12606	* the problem).
12607	*/
12608	IEMXCPTRAISE enmRaise;
12609	IEMXCPTRAISEINFO fRaiseInfo;
12610	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
12611	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
12612	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
12613	{
12614	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
12615	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12616	}
12617	else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
12618	{
12619	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12620	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
12621	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
12622	/** @todo Make AssertMsgReturn as just AssertMsg later. */
12623	AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
12624	("Unexpected VM-exit interruption vector type %#x!\n", uExitVectorType), VERR_VMX_IPE_5);
12625
12626	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
12627
12628	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
12629	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
12630	{
12631	pVmxTransient->fVectoringPF = true;
12632	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12633	}
12634	}
12635	else
12636	{
12637	/*
12638	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
12639	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
12640	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
12641	*/
12642	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12643	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12644	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
12645	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12646	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12647	}
12648
12649	/*
12650	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
12651	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
12652	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
12653	* subsequent VM-entry would fail.
12654	*
12655	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
12656	*/
12657	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS)
12658	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12659	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
12660	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
12661	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
12662	{
12663	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
12664	}
12665
12666	switch (enmRaise)
12667	{
12668	case IEMXCPTRAISE_CURRENT_XCPT:
12669	{
12670	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
12671	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
12672	Assert(rcStrict == VINF_SUCCESS);
12673	break;
12674	}
12675
12676	case IEMXCPTRAISE_PREV_EVENT:
12677	{
12678	uint32_t u32ErrCode;
12679	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
12680	{
12681	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12682	AssertRCReturn(rc2, rc2);
12683	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12684	}
12685	else
12686	u32ErrCode = 0;
12687
12688	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
12689	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
12690	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12691	0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
12692
12693	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
12694	pVCpu->hm.s.Event.u32ErrCode));
12695	Assert(rcStrict == VINF_SUCCESS);
12696	break;
12697	}
12698
12699	case IEMXCPTRAISE_REEXEC_INSTR:
12700	Assert(rcStrict == VINF_SUCCESS);
12701	break;
12702
12703	case IEMXCPTRAISE_DOUBLE_FAULT:
12704	{
12705	/*
12706	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
12707	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
12708	*/
12709	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
12710	{
12711	pVmxTransient->fVectoringDoublePF = true;
12712	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
12713	pVCpu->cpum.GstCtx.cr2));
12714	rcStrict = VINF_SUCCESS;
12715	}
12716	else
12717	{
12718	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
12719	hmR0VmxSetPendingXcptDF(pVCpu);
12720	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
12721	uIdtVector, uExitVector));
12722	rcStrict = VINF_HM_DOUBLE_FAULT;
12723	}
12724	break;
12725	}
12726
12727	case IEMXCPTRAISE_TRIPLE_FAULT:
12728	{
12729	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
12730	rcStrict = VINF_EM_RESET;
12731	break;
12732	}
12733
12734	case IEMXCPTRAISE_CPU_HANG:
12735	{
12736	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
12737	rcStrict = VERR_EM_GUEST_CPU_HANG;
12738	break;
12739	}
12740
12741	default:
12742	{
12743	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
12744	rcStrict = VERR_VMX_IPE_2;
12745	break;
12746	}
12747	}
12748	}
12749	else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
12750	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
12751	&& uExitVector != X86_XCPT_DF
12752	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
12753	{
12754	/*
12755	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
12756	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
12757	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
12758	*/
12759	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
12760	{
12761	Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
12762	VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
12763	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
12764	}
12765	}
12766
12767	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
12768	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
12769	return rcStrict;
12770	}
12771
12772
12773	/** @name VM-exit handlers.
12774	* @{
12775	*/
12776	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12777	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12778	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12779
12780	/**
12781	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
12782	*/
12783	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12784	{
12785	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12786	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
12787	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
12788	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
12789	return VINF_SUCCESS;
12790	return VINF_EM_RAW_INTERRUPT;
12791	}
12792
12793
12794	/**
12795	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
12796	*/
12797	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12798	{
12799	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12800	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
12801
12802	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12803	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12804	AssertRCReturn(rc, rc);
12805
12806	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12807	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
12808	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
12809	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
12810
12811	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12812	{
12813	/*
12814	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
12815	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
12816	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
12817	*
12818	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
12819	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
12820	*/
12821	VMXDispatchHostNmi();
12822	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
12823	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12824	return VINF_SUCCESS;
12825	}
12826
12827	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12828	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12829	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
12830	{ /* likely */ }
12831	else
12832	{
12833	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
12834	rcStrictRc1 = VINF_SUCCESS;
12835	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12836	return rcStrictRc1;
12837	}
12838
12839	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
12840	uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
12841	switch (uIntType)
12842	{
12843	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
12844	Assert(uVector == X86_XCPT_DB);
12845	RT_FALL_THRU();
12846	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
12847	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
12848	RT_FALL_THRU();
12849	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
12850	{
12851	/*
12852	* If there's any exception caused as a result of event injection, the resulting
12853	* secondary/final execption will be pending, we shall continue guest execution
12854	* after injecting the event. The page-fault case is complicated and we manually
12855	* handle any currently pending event in hmR0VmxExitXcptPF.
12856	*/
12857	if (!pVCpu->hm.s.Event.fPending)
12858	{ /* likely */ }
12859	else if (uVector != X86_XCPT_PF)
12860	{
12861	rc = VINF_SUCCESS;
12862	break;
12863	}
12864
12865	switch (uVector)
12866	{
12867	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
12868	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
12869	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
12870	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
12871	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
12872	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
12873
12874	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
12875	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12876	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
12877	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12878	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
12879	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12880	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
12881	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12882	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
12883	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12884	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
12885	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12886	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
12887	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12888	default:
12889	{
12890	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
12891	if (pVmcsInfo->RealMode.fRealOnV86Active)
12892	{
12893	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
12894	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
12895	Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
12896
12897	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
12898	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12899	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12900	AssertRCReturn(rc, rc);
12901	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
12902	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
12903	0 /* GCPtrFaultAddress */);
12904	}
12905	else
12906	{
12907	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
12908	pVCpu->hm.s.u32HMError = uVector;
12909	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
12910	}
12911	break;
12912	}
12913	}
12914	break;
12915	}
12916
12917	default:
12918	{
12919	pVCpu->hm.s.u32HMError = uExitIntInfo;
12920	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
12921	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
12922	break;
12923	}
12924	}
12925	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12926	return rc;
12927	}
12928
12929
12930	/**
12931	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
12932	*/
12933	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12934	{
12935	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12936
12937	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
12938	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12939	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
12940	AssertRCReturn(rc, rc);
12941
12942	/* Evaluate and deliver pending events and resume guest execution. */
12943	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
12944	return VINF_SUCCESS;
12945	}
12946
12947
12948	/**
12949	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
12950	*/
12951	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12952	{
12953	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12954
12955	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12956	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
12957	{
12958	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
12959	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12960	}
12961
12962	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS));
12963
12964	/*
12965	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
12966	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
12967	*/
12968	uint32_t fIntrState;
12969	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
12970	AssertRCReturn(rc, rc);
12971	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
12972	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
12973	{
12974	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
12975	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12976
12977	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
12978	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
12979	AssertRCReturn(rc, rc);
12980	}
12981
12982	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
12983	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
12984	AssertRCReturn(rc, rc);
12985
12986	/* Evaluate and deliver pending events and resume guest execution. */
12987	return VINF_SUCCESS;
12988	}
12989
12990
12991	/**
12992	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
12993	*/
12994	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12995	{
12996	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12997	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12998	}
12999
13000
13001	/**
13002	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
13003	*/
13004	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13005	{
13006	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13007	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13008	}
13009
13010
13011	/**
13012	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
13013	*/
13014	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13015	{
13016	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13017
13018	/*
13019	* Get the state we need and update the exit history entry.
13020	*/
13021	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13022	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13023	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13024	AssertRCReturn(rc, rc);
13025
13026	VBOXSTRICTRC rcStrict;
13027	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
13028	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
13029	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
13030	if (!pExitRec)
13031	{
13032	/*
13033	* Regular CPUID instruction execution.
13034	*/
13035	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
13036	if (rcStrict == VINF_SUCCESS)
13037	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13038	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13039	{
13040	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13041	rcStrict = VINF_SUCCESS;
13042	}
13043	}
13044	else
13045	{
13046	/*
13047	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
13048	*/
13049	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13050	AssertRCReturn(rc2, rc2);
13051
13052	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
13053	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
13054
13055	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
13056	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13057
13058	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
13059	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13060	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
13061	}
13062	return rcStrict;
13063	}
13064
13065
13066	/**
13067	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
13068	*/
13069	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13070	{
13071	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13072
13073	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13074	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
13075	AssertRCReturn(rc, rc);
13076
13077	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
13078	return VINF_EM_RAW_EMULATE_INSTR;
13079
13080	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
13081	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13082	}
13083
13084
13085	/**
13086	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
13087	*/
13088	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13089	{
13090	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13091
13092	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13093	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13094	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13095	AssertRCReturn(rc, rc);
13096
13097	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
13098	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13099	{
13100	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13101	we must reset offsetting on VM-entry. See @bugref{6634}. */
13102	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13103	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13104	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13105	}
13106	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13107	{
13108	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13109	rcStrict = VINF_SUCCESS;
13110	}
13111	return rcStrict;
13112	}
13113
13114
13115	/**
13116	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
13117	*/
13118	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13119	{
13120	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13121
13122	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13123	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
13124	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13125	AssertRCReturn(rc, rc);
13126
13127	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
13128	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13129	{
13130	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13131	we must reset offsetting on VM-reentry. See @bugref{6634}. */
13132	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13133	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13134	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13135	}
13136	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13137	{
13138	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13139	rcStrict = VINF_SUCCESS;
13140	}
13141	return rcStrict;
13142	}
13143
13144
13145	/**
13146	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
13147	*/
13148	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13149	{
13150	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13151
13152	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13153	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
13154	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13155	AssertRCReturn(rc, rc);
13156
13157	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13158	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13159	if (RT_LIKELY(rc == VINF_SUCCESS))
13160	{
13161	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13162	Assert(pVmxTransient->cbInstr == 2);
13163	}
13164	else
13165	{
13166	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
13167	rc = VERR_EM_INTERPRETER;
13168	}
13169	return rc;
13170	}
13171
13172
13173	/**
13174	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
13175	*/
13176	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13177	{
13178	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13179
13180	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
13181	if (EMAreHypercallInstructionsEnabled(pVCpu))
13182	{
13183	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13184	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
13185	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13186	AssertRCReturn(rc, rc);
13187
13188	/* Perform the hypercall. */
13189	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
13190	if (rcStrict == VINF_SUCCESS)
13191	{
13192	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13193	AssertRCReturn(rc, rc);
13194	}
13195	else
13196	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
13197	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
13198	\|\| RT_FAILURE(rcStrict));
13199
13200	/* If the hypercall changes anything other than guest's general-purpose registers,
13201	we would need to reload the guest changed bits here before VM-entry. */
13202	}
13203	else
13204	Log4Func(("Hypercalls not enabled\n"));
13205
13206	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
13207	if (RT_FAILURE(rcStrict))
13208	{
13209	hmR0VmxSetPendingXcptUD(pVCpu);
13210	rcStrict = VINF_SUCCESS;
13211	}
13212
13213	return rcStrict;
13214	}
13215
13216
13217	/**
13218	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
13219	*/
13220	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13221	{
13222	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13223	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
13224
13225	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13226	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13227	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13228	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13229	AssertRCReturn(rc, rc);
13230
13231	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
13232
13233	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
13234	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13235	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13236	{
13237	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13238	rcStrict = VINF_SUCCESS;
13239	}
13240	else
13241	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n", pVmxTransient->uExitQual,
13242	VBOXSTRICTRC_VAL(rcStrict)));
13243	return rcStrict;
13244	}
13245
13246
13247	/**
13248	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
13249	*/
13250	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13251	{
13252	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13253
13254	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13255	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13256	AssertRCReturn(rc, rc);
13257
13258	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13259	rc = EMInterpretMonitor(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13260	if (RT_LIKELY(rc == VINF_SUCCESS))
13261	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13262	else
13263	{
13264	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
13265	rc = VERR_EM_INTERPRETER;
13266	}
13267	return rc;
13268	}
13269
13270
13271	/**
13272	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
13273	*/
13274	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13275	{
13276	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13277
13278	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13279	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13280	AssertRCReturn(rc, rc);
13281
13282	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13283	VBOXSTRICTRC rc2 = EMInterpretMWait(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13284	rc = VBOXSTRICTRC_VAL(rc2);
13285	if (RT_LIKELY( rc == VINF_SUCCESS
13286	\|\| rc == VINF_EM_HALT))
13287	{
13288	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13289	AssertRCReturn(rc3, rc3);
13290
13291	if ( rc == VINF_EM_HALT
13292	&& EMMonitorWaitShouldContinue(pVCpu, pCtx))
13293	rc = VINF_SUCCESS;
13294	}
13295	else
13296	{
13297	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
13298	rc = VERR_EM_INTERPRETER;
13299	}
13300	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
13301	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
13302	return rc;
13303	}
13304
13305
13306	/**
13307	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
13308	*/
13309	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13310	{
13311	/*
13312	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
13313	* mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
13314	* treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
13315	* VMX root operation. If we get here, something funny is going on.
13316	*
13317	* See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
13318	*/
13319	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13320	AssertMsgFailed(("Unexpected RSM VM-exit\n"));
13321	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13322	}
13323
13324
13325	/**
13326	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
13327	*/
13328	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13329	{
13330	/*
13331	* This can only happen if we support dual-monitor treatment of SMI, which can be activated
13332	* by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
13333	* this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
13334	* an SMI. If we get here, something funny is going on.
13335	*
13336	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
13337	* See Intel spec. 25.3 "Other Causes of VM-Exits"
13338	*/
13339	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13340	AssertMsgFailed(("Unexpected SMI VM-exit\n"));
13341	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13342	}
13343
13344
13345	/**
13346	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
13347	*/
13348	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13349	{
13350	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
13351	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13352	AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
13353	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13354	}
13355
13356
13357	/**
13358	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
13359	*/
13360	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13361	{
13362	/*
13363	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
13364	* We don't make use of it as our guests don't have direct access to the host LAPIC.
13365	* See Intel spec. 25.3 "Other Causes of VM-exits".
13366	*/
13367	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13368	AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
13369	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13370	}
13371
13372
13373	/**
13374	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
13375	* VM-exit.
13376	*/
13377	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13378	{
13379	/*
13380	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
13381	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
13382	*
13383	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
13384	* See Intel spec. "23.8 Restrictions on VMX operation".
13385	*/
13386	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13387	return VINF_SUCCESS;
13388	}
13389
13390
13391	/**
13392	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
13393	* VM-exit.
13394	*/
13395	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13396	{
13397	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13398	return VINF_EM_RESET;
13399	}
13400
13401
13402	/**
13403	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
13404	*/
13405	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13406	{
13407	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13408
13409	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13410	AssertRCReturn(rc, rc);
13411
13412	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
13413	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
13414	rc = VINF_SUCCESS;
13415	else
13416	rc = VINF_EM_HALT;
13417
13418	if (rc != VINF_SUCCESS)
13419	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
13420	return rc;
13421	}
13422
13423
13424	/**
13425	* VM-exit handler for instructions that result in a \#UD exception delivered to
13426	* the guest.
13427	*/
13428	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13429	{
13430	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13431	hmR0VmxSetPendingXcptUD(pVCpu);
13432	return VINF_SUCCESS;
13433	}
13434
13435
13436	/**
13437	* VM-exit handler for expiry of the VMX-preemption timer.
13438	*/
13439	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13440	{
13441	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13442
13443	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
13444	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13445
13446	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
13447	PVM pVM = pVCpu->CTX_SUFF(pVM);
13448	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
13449	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
13450	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
13451	}
13452
13453
13454	/**
13455	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
13456	*/
13457	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13458	{
13459	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13460
13461	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13462	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13463	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
13464	AssertRCReturn(rc, rc);
13465
13466	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
13467	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13468	: HM_CHANGED_RAISED_XCPT_MASK);
13469
13470	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13471	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
13472
13473	return rcStrict;
13474	}
13475
13476
13477	/**
13478	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
13479	*/
13480	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13481	{
13482	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13483	/** @todo Use VM-exit instruction information. */
13484	return VERR_EM_INTERPRETER;
13485	}
13486
13487
13488	/**
13489	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
13490	* Error VM-exit.
13491	*/
13492	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13493	{
13494	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13495	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13496	AssertRCReturn(rc, rc);
13497
13498	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo);
13499	if (RT_FAILURE(rc))
13500	return rc;
13501
13502	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
13503	NOREF(uInvalidReason);
13504
13505	#ifdef VBOX_STRICT
13506	uint32_t fIntrState;
13507	RTHCUINTREG uHCReg;
13508	uint64_t u64Val;
13509	uint32_t u32Val;
13510	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
13511	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
13512	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
13513	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13514	AssertRCReturn(rc, rc);
13515
13516	Log4(("uInvalidReason %u\n", uInvalidReason));
13517	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
13518	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
13519	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
13520	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
13521
13522	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
13523	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
13524	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
13525	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
13526	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
13527	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13528	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
13529	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
13530	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
13531	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13532	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
13533	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
13534
13535	hmR0DumpRegs(pVCpu);
13536	#endif
13537
13538	return VERR_VMX_INVALID_GUEST_STATE;
13539	}
13540
13541
13542	/**
13543	* VM-exit handler for VM-entry failure due to an MSR-load
13544	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
13545	*/
13546	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13547	{
13548	AssertMsgFailed(("Unexpected MSR-load exit\n"));
13549	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13550	}
13551
13552
13553	/**
13554	* VM-exit handler for VM-entry failure due to a machine-check event
13555	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
13556	*/
13557	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13558	{
13559	AssertMsgFailed(("Unexpected machine-check event exit\n"));
13560	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13561	}
13562
13563
13564	/**
13565	* VM-exit handler for all undefined reasons. Should never ever happen.. in
13566	* theory.
13567	*/
13568	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13569	{
13570	RT_NOREF2(pVCpu, pVmxTransient);
13571	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
13572	return VERR_VMX_UNDEFINED_EXIT_CODE;
13573	}
13574
13575
13576	/**
13577	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
13578	* (VMX_EXIT_GDTR_IDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
13579	* Conditional VM-exit.
13580	*/
13581	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13582	{
13583	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13584
13585	/* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
13586	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
13587	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13588	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
13589	return VERR_EM_INTERPRETER;
13590	AssertMsgFailed(("Unexpected XDTR access\n"));
13591	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13592	}
13593
13594
13595	/**
13596	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
13597	*/
13598	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13599	{
13600	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13601
13602	/* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
13603	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13604	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
13605	return VERR_EM_INTERPRETER;
13606	AssertMsgFailed(("Unexpected RDRAND exit\n"));
13607	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13608	}
13609
13610
13611	/**
13612	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
13613	*/
13614	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13615	{
13616	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13617
13618	/** @todo Optimize this: We currently drag in in the whole MSR state
13619	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13620	* MSRs required. That would require changes to IEM and possibly CPUM too.
13621	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13622	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13623	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13624	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13625	switch (idMsr)
13626	{
13627	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13628	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13629	}
13630
13631	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13632	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13633	AssertRCReturn(rc, rc);
13634
13635	Log4Func(("ecx=%#RX32\n", idMsr));
13636
13637	#ifdef VBOX_STRICT
13638	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
13639	{
13640	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
13641	&& idMsr != MSR_K6_EFER)
13642	{
13643	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
13644	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13645	}
13646	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13647	{
13648	Assert(pVmcsInfo->pvMsrBitmap);
13649	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13650	if (fMsrpm & VMXMSRPM_ALLOW_RD)
13651	{
13652	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
13653	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13654	}
13655	}
13656	}
13657	#endif
13658
13659	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
13660	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
13661	if (rcStrict == VINF_SUCCESS)
13662	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13663	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
13664	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13665	{
13666	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13667	rcStrict = VINF_SUCCESS;
13668	}
13669	else
13670	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13671
13672	return rcStrict;
13673	}
13674
13675
13676	/**
13677	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
13678	*/
13679	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13680	{
13681	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13682
13683	/** @todo Optimize this: We currently drag in in the whole MSR state
13684	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13685	* MSRs required. That would require changes to IEM and possibly CPUM too.
13686	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13687	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13688	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13689
13690	/*
13691	* The FS and GS base MSRs are not part of the above all-MSRs mask.
13692	* Although we don't need to fetch the base as it will be overwritten shortly, while
13693	* loading guest-state we would also load the entire segment register including limit
13694	* and attributes and thus we need to load them here.
13695	*/
13696	switch (idMsr)
13697	{
13698	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13699	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13700	}
13701
13702	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13703	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13704	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13705	AssertRCReturn(rc, rc);
13706
13707	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
13708
13709	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
13710	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
13711
13712	if (rcStrict == VINF_SUCCESS)
13713	{
13714	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13715
13716	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
13717	if ( idMsr == MSR_IA32_APICBASE
13718	\|\| ( idMsr >= MSR_IA32_X2APIC_START
13719	&& idMsr <= MSR_IA32_X2APIC_END))
13720	{
13721	/*
13722	* We've already saved the APIC related guest-state (TPR) in post-run phase.
13723	* When full APIC register virtualization is implemented we'll have to make
13724	* sure APIC state is saved from the VMCS before IEM changes it.
13725	*/
13726	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
13727	}
13728	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
13729	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13730	else if (idMsr == MSR_K6_EFER)
13731	{
13732	/*
13733	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
13734	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
13735	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
13736	*/
13737	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
13738	}
13739
13740	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
13741	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
13742	{
13743	switch (idMsr)
13744	{
13745	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
13746	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
13747	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
13748	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
13749	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
13750	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
13751	default:
13752	{
13753	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
13754	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
13755	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13756	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
13757	break;
13758	}
13759	}
13760	}
13761	#ifdef VBOX_STRICT
13762	else
13763	{
13764	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
13765	switch (idMsr)
13766	{
13767	case MSR_IA32_SYSENTER_CS:
13768	case MSR_IA32_SYSENTER_EIP:
13769	case MSR_IA32_SYSENTER_ESP:
13770	case MSR_K8_FS_BASE:
13771	case MSR_K8_GS_BASE:
13772	{
13773	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
13774	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13775	}
13776
13777	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
13778	default:
13779	{
13780	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
13781	{
13782	/* EFER MSR writes are always intercepted. */
13783	if (idMsr != MSR_K6_EFER)
13784	{
13785	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
13786	idMsr));
13787	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13788	}
13789	}
13790
13791	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13792	{
13793	Assert(pVmcsInfo->pvMsrBitmap);
13794	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13795	if (fMsrpm & VMXMSRPM_ALLOW_WR)
13796	{
13797	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
13798	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13799	}
13800	}
13801	break;
13802	}
13803	}
13804	}
13805	#endif /* VBOX_STRICT */
13806	}
13807	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13808	{
13809	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13810	rcStrict = VINF_SUCCESS;
13811	}
13812	else
13813	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13814
13815	return rcStrict;
13816	}
13817
13818
13819	/**
13820	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
13821	*/
13822	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13823	{
13824	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13825	/** @todo The guest has likely hit a contended spinlock. We might want to
13826	* poke a schedule different guest VCPU. */
13827	return VINF_EM_RAW_INTERRUPT;
13828	}
13829
13830
13831	/**
13832	* VM-exit handler for when the TPR value is lowered below the specified
13833	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
13834	*/
13835	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13836	{
13837	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13838	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
13839
13840	/*
13841	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
13842	* We'll re-evaluate pending interrupts and inject them before the next VM
13843	* entry so we can just continue execution here.
13844	*/
13845	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
13846	return VINF_SUCCESS;
13847	}
13848
13849
13850	/**
13851	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
13852	* VM-exit.
13853	*
13854	* @retval VINF_SUCCESS when guest execution can continue.
13855	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
13856	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
13857	* interpreter.
13858	*/
13859	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13860	{
13861	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13862	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
13863
13864	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13865	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13866	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13867	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13868	AssertRCReturn(rc, rc);
13869
13870	VBOXSTRICTRC rcStrict;
13871	PVM pVM = pVCpu->CTX_SUFF(pVM);
13872	RTGCUINTPTR const uExitQual = pVmxTransient->uExitQual;
13873	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
13874	switch (uAccessType)
13875	{
13876	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
13877	{
13878	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
13879	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
13880	VMX_EXIT_QUAL_CRX_GENREG(uExitQual));
13881	AssertMsg( rcStrict == VINF_SUCCESS
13882	\|\| rcStrict == VINF_IEM_RAISED_XCPT
13883	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13884
13885	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
13886	{
13887	case 0:
13888	{
13889	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13890	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13891	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
13892	Log4Func(("CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
13893
13894	/*
13895	* This is a kludge for handling switches back to real mode when we try to use
13896	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
13897	* deal with special selector values, so we have to return to ring-3 and run
13898	* there till the selector values are V86 mode compatible.
13899	*
13900	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
13901	* latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
13902	* at the end of this function.
13903	*/
13904	if ( rc == VINF_SUCCESS
13905	&& !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
13906	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
13907	&& (uOldCr0 & X86_CR0_PE)
13908	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
13909	{
13910	/** @todo check selectors rather than returning all the time. */
13911	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
13912	rcStrict = VINF_EM_RESCHEDULE_REM;
13913	}
13914	break;
13915	}
13916
13917	case 2:
13918	{
13919	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
13920	/* Nothing to do here, CR2 it's not part of the VMCS. */
13921	break;
13922	}
13923
13924	case 3:
13925	{
13926	Assert( !pVM->hm.s.fNestedPaging
13927	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
13928	\|\| pVCpu->hm.s.fUsingDebugLoop);
13929	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
13930	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13931	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
13932	Log4Func(("CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
13933	break;
13934	}
13935
13936	case 4:
13937	{
13938	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
13939	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13940	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
13941	Log4Func(("CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
13942	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
13943	break;
13944	}
13945
13946	case 8:
13947	{
13948	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
13949	Assert(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
13950	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13951	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
13952	break;
13953	}
13954	default:
13955	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual)));
13956	break;
13957	}
13958	break;
13959	}
13960
13961	case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
13962	{
13963	Assert( !pVM->hm.s.fNestedPaging
13964	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
13965	\|\| pVCpu->hm.s.fUsingDebugLoop
13966	\|\| VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 3);
13967	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
13968	Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 8
13969	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
13970
13971	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_GENREG(uExitQual),
13972	VMX_EXIT_QUAL_CRX_REGISTER(uExitQual));
13973	AssertMsg( rcStrict == VINF_SUCCESS
13974	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13975	#ifdef VBOX_WITH_STATISTICS
13976	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
13977	{
13978	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
13979	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
13980	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
13981	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
13982	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
13983	}
13984	#endif
13985	Log4Func(("CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
13986	VBOXSTRICTRC_VAL(rcStrict)));
13987	if (VMX_EXIT_QUAL_CRX_GENREG(uExitQual) == X86_GREG_xSP)
13988	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
13989	else
13990	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13991	break;
13992	}
13993
13994	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
13995	{
13996	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
13997	AssertMsg( rcStrict == VINF_SUCCESS
13998	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13999
14000	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14001	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
14002	Log4Func(("CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
14003	break;
14004	}
14005
14006	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
14007	{
14008	/* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
14009	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
14010	AssertRCReturn(rc, rc);
14011	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual),
14012	pVmxTransient->uGuestLinearAddr);
14013	AssertMsg( rcStrict == VINF_SUCCESS
14014	\|\| rcStrict == VINF_IEM_RAISED_XCPT
14015	, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14016
14017	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14018	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
14019	Log4Func(("LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
14020	break;
14021	}
14022
14023	default:
14024	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
14025	VERR_VMX_UNEXPECTED_EXCEPTION);
14026	}
14027
14028	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
14029	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
14030	if (rcStrict == VINF_IEM_RAISED_XCPT)
14031	{
14032	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14033	rcStrict = VINF_SUCCESS;
14034	}
14035
14036	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
14037	NOREF(pVM);
14038	return rcStrict;
14039	}
14040
14041
14042	/**
14043	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
14044	* VM-exit.
14045	*/
14046	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14047	{
14048	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14049	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
14050
14051	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14052	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14053	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14054	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14055	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
14056	\| CPUMCTX_EXTRN_EFER);
14057	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
14058	AssertRCReturn(rc, rc);
14059
14060	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
14061	uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
14062	uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQual);
14063	bool fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
14064	bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
14065	bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
14066	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
14067	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
14068
14069	/*
14070	* Update exit history to see if this exit can be optimized.
14071	*/
14072	VBOXSTRICTRC rcStrict;
14073	PCEMEXITREC pExitRec = NULL;
14074	if ( !fGstStepping
14075	&& !fDbgStepping)
14076	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14077	!fIOString
14078	? !fIOWrite
14079	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
14080	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
14081	: !fIOWrite
14082	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
14083	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
14084	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14085	if (!pExitRec)
14086	{
14087	/* I/O operation lookup arrays. */
14088	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
14089	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
14090	uint32_t const cbValue = s_aIOSizes[uIOWidth];
14091	uint32_t const cbInstr = pVmxTransient->cbInstr;
14092	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
14093	PVM pVM = pVCpu->CTX_SUFF(pVM);
14094	if (fIOString)
14095	{
14096	/*
14097	* INS/OUTS - I/O String instruction.
14098	*
14099	* Use instruction-information if available, otherwise fall back on
14100	* interpreting the instruction.
14101	*/
14102	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14103	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
14104	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
14105	if (fInsOutsInfo)
14106	{
14107	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14108	AssertRCReturn(rc2, rc2);
14109	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
14110	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
14111	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
14112	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
14113	if (fIOWrite)
14114	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
14115	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
14116	else
14117	{
14118	/*
14119	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
14120	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
14121	* See Intel Instruction spec. for "INS".
14122	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
14123	*/
14124	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
14125	}
14126	}
14127	else
14128	rcStrict = IEMExecOne(pVCpu);
14129
14130	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14131	fUpdateRipAlready = true;
14132	}
14133	else
14134	{
14135	/*
14136	* IN/OUT - I/O instruction.
14137	*/
14138	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14139	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
14140	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
14141	if (fIOWrite)
14142	{
14143	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
14144	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
14145	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14146	&& !pCtx->eflags.Bits.u1TF)
14147	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
14148	}
14149	else
14150	{
14151	uint32_t u32Result = 0;
14152	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
14153	if (IOM_SUCCESS(rcStrict))
14154	{
14155	/* Save result of I/O IN instr. in AL/AX/EAX. */
14156	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
14157	}
14158	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14159	&& !pCtx->eflags.Bits.u1TF)
14160	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
14161	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
14162	}
14163	}
14164
14165	if (IOM_SUCCESS(rcStrict))
14166	{
14167	if (!fUpdateRipAlready)
14168	{
14169	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
14170	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14171	}
14172
14173	/*
14174	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
14175	* while booting Fedora 17 64-bit guest.
14176	*
14177	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
14178	*/
14179	if (fIOString)
14180	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
14181
14182	/*
14183	* If any I/O breakpoints are armed, we need to check if one triggered
14184	* and take appropriate action.
14185	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
14186	*/
14187	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
14188	AssertRCReturn(rc, rc);
14189
14190	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
14191	* execution engines about whether hyper BPs and such are pending. */
14192	uint32_t const uDr7 = pCtx->dr[7];
14193	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
14194	&& X86_DR7_ANY_RW_IO(uDr7)
14195	&& (pCtx->cr4 & X86_CR4_DE))
14196	\|\| DBGFBpIsHwIoArmed(pVM)))
14197	{
14198	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
14199
14200	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
14201	VMMRZCallRing3Disable(pVCpu);
14202	HM_DISABLE_PREEMPT(pVCpu);
14203
14204	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
14205
14206	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
14207	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
14208	{
14209	/* Raise #DB. */
14210	if (fIsGuestDbgActive)
14211	ASMSetDR6(pCtx->dr[6]);
14212	if (pCtx->dr[7] != uDr7)
14213	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
14214
14215	hmR0VmxSetPendingXcptDB(pVCpu);
14216	}
14217	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
14218	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
14219	else if ( rcStrict2 != VINF_SUCCESS
14220	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
14221	rcStrict = rcStrict2;
14222	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
14223
14224	HM_RESTORE_PREEMPT();
14225	VMMRZCallRing3Enable(pVCpu);
14226	}
14227	}
14228
14229	#ifdef VBOX_STRICT
14230	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14231	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
14232	Assert(!fIOWrite);
14233	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14234	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
14235	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
14236	Assert(fIOWrite);
14237	else
14238	{
14239	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
14240	* statuses, that the VMM device and some others may return. See
14241	* IOM_SUCCESS() for guidance. */
14242	AssertMsg( RT_FAILURE(rcStrict)
14243	\|\| rcStrict == VINF_SUCCESS
14244	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
14245	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
14246	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
14247	\|\| rcStrict == VINF_EM_RAW_TO_R3
14248	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14249	# endif
14250	}
14251	#endif
14252	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
14253	}
14254	else
14255	{
14256	/*
14257	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14258	*/
14259	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14260	AssertRCReturn(rc2, rc2);
14261	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
14262	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
14263	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
14264	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14265	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
14266	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
14267
14268	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14269	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14270
14271	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14272	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14273	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14274	}
14275	return rcStrict;
14276	}
14277
14278
14279	/**
14280	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
14281	* VM-exit.
14282	*/
14283	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14284	{
14285	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14286
14287	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
14288	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14289	AssertRCReturn(rc, rc);
14290	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
14291	{
14292	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14293	AssertRCReturn(rc, rc);
14294	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
14295	{
14296	uint32_t uErrCode;
14297	RTGCUINTPTR GCPtrFaultAddress;
14298	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
14299	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
14300	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
14301	if (fErrorCodeValid)
14302	{
14303	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14304	AssertRCReturn(rc, rc);
14305	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
14306	}
14307	else
14308	uErrCode = 0;
14309
14310	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
14311	&& uVector == X86_XCPT_PF)
14312	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
14313	else
14314	GCPtrFaultAddress = 0;
14315
14316	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14317	AssertRCReturn(rc, rc);
14318
14319	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
14320	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
14321
14322	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
14323	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14324	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14325	}
14326	}
14327
14328	/* Fall back to the interpreter to emulate the task-switch. */
14329	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14330	return VERR_EM_INTERPRETER;
14331	}
14332
14333
14334	/**
14335	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
14336	*/
14337	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14338	{
14339	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14340
14341	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14342	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
14343	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14344	AssertRCReturn(rc, rc);
14345	return VINF_EM_DBG_STEPPED;
14346	}
14347
14348
14349	/**
14350	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
14351	*/
14352	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14353	{
14354	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14355	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
14356
14357	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14358	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14359	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14360	{
14361	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
14362	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14363	{
14364	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14365	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14366	}
14367	}
14368	else
14369	{
14370	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14371	rcStrict1 = VINF_SUCCESS;
14372	return rcStrict1;
14373	}
14374
14375	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
14376	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14377	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14378	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14379	AssertRCReturn(rc, rc);
14380
14381	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
14382	uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
14383	VBOXSTRICTRC rcStrict2;
14384	switch (uAccessType)
14385	{
14386	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
14387	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
14388	{
14389	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
14390	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
14391	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
14392
14393	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
14394	GCPhys &= PAGE_BASE_GC_MASK;
14395	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
14396	PVM pVM = pVCpu->CTX_SUFF(pVM);
14397	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
14398	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
14399
14400	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14401	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
14402	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
14403	CPUMCTX2CORE(pCtx), GCPhys);
14404	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14405	if ( rcStrict2 == VINF_SUCCESS
14406	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14407	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14408	{
14409	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14410	\| HM_CHANGED_GUEST_APIC_TPR);
14411	rcStrict2 = VINF_SUCCESS;
14412	}
14413	break;
14414	}
14415
14416	default:
14417	Log4Func(("uAccessType=%#x\n", uAccessType));
14418	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
14419	break;
14420	}
14421
14422	if (rcStrict2 != VINF_SUCCESS)
14423	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
14424	return rcStrict2;
14425	}
14426
14427
14428	/**
14429	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
14430	* VM-exit.
14431	*/
14432	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14433	{
14434	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14435
14436	/* We should -not- get this VM-exit if the guest's debug registers were active. */
14437	if (pVmxTransient->fWasGuestDebugStateActive)
14438	{
14439	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
14440	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14441	}
14442
14443	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14444	if ( !pVCpu->hm.s.fSingleInstruction
14445	&& !pVmxTransient->fWasHyperDebugStateActive)
14446	{
14447	Assert(!DBGFIsStepping(pVCpu));
14448	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
14449
14450	/* Don't intercept MOV DRx any more. */
14451	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
14452	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14453	AssertRCReturn(rc, rc);
14454
14455	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
14456	VMMRZCallRing3Disable(pVCpu);
14457	HM_DISABLE_PREEMPT(pVCpu);
14458
14459	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
14460	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
14461	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
14462
14463	HM_RESTORE_PREEMPT();
14464	VMMRZCallRing3Enable(pVCpu);
14465
14466	#ifdef VBOX_WITH_STATISTICS
14467	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14468	AssertRCReturn(rc, rc);
14469	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14470	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14471	else
14472	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14473	#endif
14474	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
14475	return VINF_SUCCESS;
14476	}
14477
14478	/*
14479	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
14480	* The EFER MSR is always up-to-date.
14481	* Update the segment registers and DR7 from the CPU.
14482	*/
14483	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14484	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14485	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
14486	AssertRCReturn(rc, rc);
14487	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
14488
14489	PVM pVM = pVCpu->CTX_SUFF(pVM);
14490	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14491	{
14492	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14493	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
14494	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
14495	if (RT_SUCCESS(rc))
14496	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
14497	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14498	}
14499	else
14500	{
14501	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14502	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
14503	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
14504	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14505	}
14506
14507	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
14508	if (RT_SUCCESS(rc))
14509	{
14510	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14511	AssertRCReturn(rc2, rc2);
14512	return VINF_SUCCESS;
14513	}
14514	return rc;
14515	}
14516
14517
14518	/**
14519	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
14520	* Conditional VM-exit.
14521	*/
14522	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14523	{
14524	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14525	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14526
14527	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14528	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14529	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14530	{
14531	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
14532	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
14533	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14534	{
14535	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14536	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14537	}
14538	}
14539	else
14540	{
14541	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14542	rcStrict1 = VINF_SUCCESS;
14543	return rcStrict1;
14544	}
14545
14546	/*
14547	* Get sufficent state and update the exit history entry.
14548	*/
14549	RTGCPHYS GCPhys;
14550	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14551	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14552	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14553	AssertRCReturn(rc, rc);
14554
14555	VBOXSTRICTRC rcStrict;
14556	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14557	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
14558	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14559	if (!pExitRec)
14560	{
14561	/*
14562	* If we succeed, resume guest execution.
14563	* If we fail in interpreting the instruction because we couldn't get the guest physical address
14564	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
14565	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
14566	* weird case. See @bugref{6043}.
14567	*/
14568	PVM pVM = pVCpu->CTX_SUFF(pVM);
14569	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14570	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
14571	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
14572	if ( rcStrict == VINF_SUCCESS
14573	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
14574	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
14575	{
14576	/* Successfully handled MMIO operation. */
14577	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14578	\| HM_CHANGED_GUEST_APIC_TPR);
14579	rcStrict = VINF_SUCCESS;
14580	}
14581	}
14582	else
14583	{
14584	/*
14585	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14586	*/
14587	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14588	AssertRCReturn(rc2, rc2);
14589
14590	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
14591	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
14592
14593	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14594	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14595
14596	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14597	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14598	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14599	}
14600	return VBOXSTRICTRC_TODO(rcStrict);
14601	}
14602
14603
14604	/**
14605	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
14606	* VM-exit.
14607	*/
14608	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14609	{
14610	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14611	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14612
14613	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14614	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14615	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14616	{
14617	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
14618	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14619	Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
14620	}
14621	else
14622	{
14623	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14624	rcStrict1 = VINF_SUCCESS;
14625	return rcStrict1;
14626	}
14627
14628	RTGCPHYS GCPhys;
14629	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14630	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14631	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14632	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14633	AssertRCReturn(rc, rc);
14634
14635	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
14636	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
14637
14638	RTGCUINT uErrorCode = 0;
14639	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
14640	uErrorCode \|= X86_TRAP_PF_ID;
14641	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
14642	uErrorCode \|= X86_TRAP_PF_RW;
14643	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
14644	uErrorCode \|= X86_TRAP_PF_P;
14645
14646	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
14647
14648
14649	/* Handle the pagefault trap for the nested shadow table. */
14650	PVM pVM = pVCpu->CTX_SUFF(pVM);
14651	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14652
14653	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
14654	pCtx->cs.Sel, pCtx->rip));
14655
14656	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
14657	TRPMResetTrap(pVCpu);
14658
14659	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
14660	if ( rcStrict2 == VINF_SUCCESS
14661	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14662	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14663	{
14664	/* Successfully synced our nested page tables. */
14665	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
14666	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
14667	return VINF_SUCCESS;
14668	}
14669
14670	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14671	return rcStrict2;
14672	}
14673
14674	/** @} */
14675
14676	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14677	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14678	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14679
14680	/**
14681	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
14682	*/
14683	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14684	{
14685	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14686	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
14687
14688	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
14689	AssertRCReturn(rc, rc);
14690
14691	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
14692	{
14693	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
14694	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
14695
14696	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
14697	* provides VM-exit instruction length. If this causes problem later,
14698	* disassemble the instruction like it's done on AMD-V. */
14699	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14700	AssertRCReturn(rc2, rc2);
14701	return rc;
14702	}
14703
14704	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14705	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14706	return rc;
14707	}
14708
14709
14710	/**
14711	* VM-exit exception handler for \#BP (Breakpoint exception).
14712	*/
14713	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14714	{
14715	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14716	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
14717
14718	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14719	AssertRCReturn(rc, rc);
14720
14721	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14722	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14723	if (rc == VINF_EM_RAW_GUEST_TRAP)
14724	{
14725	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14726	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14727	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14728	AssertRCReturn(rc, rc);
14729
14730	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14731	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14732	}
14733
14734	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
14735	return rc;
14736	}
14737
14738
14739	/**
14740	* VM-exit exception handler for \#AC (alignment check exception).
14741	*/
14742	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14743	{
14744	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14745
14746	/*
14747	* Re-inject it. We'll detect any nesting before getting here.
14748	*/
14749	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14750	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14751	AssertRCReturn(rc, rc);
14752	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
14753
14754	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14755	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14756	return VINF_SUCCESS;
14757	}
14758
14759
14760	/**
14761	* VM-exit exception handler for \#DB (Debug exception).
14762	*/
14763	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14764	{
14765	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14766	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
14767
14768	/*
14769	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
14770	* for processing.
14771	*/
14772	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14773
14774	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
14775	uint64_t const uDR6 = X86_DR6_INIT_VAL
14776	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
14777	\| X86_DR6_BD \| X86_DR6_BS));
14778
14779	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14780	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
14781	Log6Func(("rc=%Rrc\n", rc));
14782	if (rc == VINF_EM_RAW_GUEST_TRAP)
14783	{
14784	/*
14785	* The exception was for the guest. Update DR6, DR7.GD and
14786	* IA32_DEBUGCTL.LBR before forwarding it.
14787	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
14788	*/
14789	VMMRZCallRing3Disable(pVCpu);
14790	HM_DISABLE_PREEMPT(pVCpu);
14791
14792	pCtx->dr[6] &= ~X86_DR6_B_MASK;
14793	pCtx->dr[6] \|= uDR6;
14794	if (CPUMIsGuestDebugStateActive(pVCpu))
14795	ASMSetDR6(pCtx->dr[6]);
14796
14797	HM_RESTORE_PREEMPT();
14798	VMMRZCallRing3Enable(pVCpu);
14799
14800	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
14801	AssertRCReturn(rc, rc);
14802
14803	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
14804	pCtx->dr[7] &= ~X86_DR7_GD;
14805
14806	/* Paranoia. */
14807	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
14808	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
14809
14810	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
14811	AssertRCReturn(rc, rc);
14812
14813	/*
14814	* Raise #DB in the guest.
14815	*
14816	* It is important to reflect exactly what the VM-exit gave us (preserving the
14817	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
14818	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
14819	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
14820	*
14821	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
14822	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
14823	*/
14824	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14825	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14826	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14827	AssertRCReturn(rc, rc);
14828	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14829	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14830	return VINF_SUCCESS;
14831	}
14832
14833	/*
14834	* Not a guest trap, must be a hypervisor related debug event then.
14835	* Update DR6 in case someone is interested in it.
14836	*/
14837	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
14838	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
14839	CPUMSetHyperDR6(pVCpu, uDR6);
14840
14841	return rc;
14842	}
14843
14844
14845	/**
14846	* Hacks its way around the lovely mesa driver's backdoor accesses.
14847	*
14848	* @sa hmR0SvmHandleMesaDrvGp
14849	*/
14850	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14851	{
14852	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
14853	RT_NOREF(pCtx);
14854
14855	/* For now we'll just skip the instruction. */
14856	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14857	}
14858
14859
14860	/**
14861	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
14862	* backdoor logging w/o checking what it is running inside.
14863	*
14864	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
14865	* backdoor port and magic numbers loaded in registers.
14866	*
14867	* @returns true if it is, false if it isn't.
14868	* @sa hmR0SvmIsMesaDrvGp
14869	*/
14870	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14871	{
14872	/* 0xed: IN eAX,dx */
14873	uint8_t abInstr[1];
14874	if (pVmxTransient->cbInstr != sizeof(abInstr))
14875	return false;
14876
14877	/* Check that it is #GP(0). */
14878	if (pVmxTransient->uExitIntErrorCode != 0)
14879	return false;
14880
14881	/* Check magic and port. */
14882	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
14883	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
14884	if (pCtx->rax != UINT32_C(0x564d5868))
14885	return false;
14886	if (pCtx->dx != UINT32_C(0x5658))
14887	return false;
14888
14889	/* Flat ring-3 CS. */
14890	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
14891	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
14892	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
14893	if (pCtx->cs.Attr.n.u2Dpl != 3)
14894	return false;
14895	if (pCtx->cs.u64Base != 0)
14896	return false;
14897
14898	/* Check opcode. */
14899	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
14900	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
14901	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
14902	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
14903	if (RT_FAILURE(rc))
14904	return false;
14905	if (abInstr[0] != 0xed)
14906	return false;
14907
14908	return true;
14909	}
14910
14911
14912	/**
14913	* VM-exit exception handler for \#GP (General-protection exception).
14914	*
14915	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
14916	*/
14917	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14918	{
14919	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14920	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
14921
14922	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14923	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14924	if (pVmcsInfo->RealMode.fRealOnV86Active)
14925	{ /* likely */ }
14926	else
14927	{
14928	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14929	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv);
14930	#endif
14931	/* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
14932	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14933	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14934	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14935	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14936	AssertRCReturn(rc, rc);
14937	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
14938	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
14939
14940	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
14941	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
14942	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14943	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14944	else
14945	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
14946	return rc;
14947	}
14948
14949	Assert(CPUMIsGuestInRealModeEx(pCtx));
14950	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
14951
14952	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14953	AssertRCReturn(rc, rc);
14954
14955	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
14956	if (rcStrict == VINF_SUCCESS)
14957	{
14958	if (!CPUMIsGuestInRealModeEx(pCtx))
14959	{
14960	/*
14961	* The guest is no longer in real-mode, check if we can continue executing the
14962	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
14963	*/
14964	pVmcsInfo->RealMode.fRealOnV86Active = false;
14965	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
14966	{
14967	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
14968	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14969	}
14970	else
14971	{
14972	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
14973	rcStrict = VINF_EM_RESCHEDULE;
14974	}
14975	}
14976	else
14977	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14978	}
14979	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14980	{
14981	rcStrict = VINF_SUCCESS;
14982	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14983	}
14984	return VBOXSTRICTRC_VAL(rcStrict);
14985	}
14986
14987
14988	/**
14989	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
14990	* the exception reported in the VMX transient structure back into the VM.
14991	*
14992	* @remarks Requires uExitIntInfo in the VMX transient structure to be
14993	* up-to-date.
14994	*/
14995	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14996	{
14997	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14998	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14999	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15000	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active,
15001	("uVector=%#x u32XcptBitmap=%#X32\n",
15002	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
15003	NOREF(pVmcsInfo);
15004	#endif
15005
15006	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
15007	hmR0VmxCheckExitDueToEventDelivery(). */
15008	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15009	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15010	AssertRCReturn(rc, rc);
15011	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15012
15013	#ifdef DEBUG_ramshankar
15014	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
15015	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n",
15016	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pCtx->cs.Sel, pCtx->rip));
15017	#endif
15018
15019	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15020	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15021	return VINF_SUCCESS;
15022	}
15023
15024
15025	/**
15026	* VM-exit exception handler for \#PF (Page-fault exception).
15027	*/
15028	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15029	{
15030	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15031	PVM pVM = pVCpu->CTX_SUFF(pVM);
15032	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15033	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15034	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15035	AssertRCReturn(rc, rc);
15036
15037	if (!pVM->hm.s.fNestedPaging)
15038	{ /* likely */ }
15039	else
15040	{
15041	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
15042	Assert(pVCpu->hm.s.fUsingDebugLoop);
15043	#endif
15044	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
15045	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
15046	{
15047	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15048	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
15049	}
15050	else
15051	{
15052	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15053	hmR0VmxSetPendingXcptDF(pVCpu);
15054	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
15055	}
15056	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15057	return rc;
15058	}
15059
15060	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
15061	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
15062	if (pVmxTransient->fVectoringPF)
15063	{
15064	Assert(pVCpu->hm.s.Event.fPending);
15065	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15066	}
15067
15068	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15069	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15070	AssertRCReturn(rc, rc);
15071
15072	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
15073	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
15074
15075	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
15076	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
15077
15078	Log4Func(("#PF: rc=%Rrc\n", rc));
15079	if (rc == VINF_SUCCESS)
15080	{
15081	/*
15082	* This is typically a shadow page table sync or a MMIO instruction. But we may have
15083	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
15084	*/
15085	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15086	TRPMResetTrap(pVCpu);
15087	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
15088	return rc;
15089	}
15090
15091	if (rc == VINF_EM_RAW_GUEST_TRAP)
15092	{
15093	if (!pVmxTransient->fVectoringDoublePF)
15094	{
15095	/* It's a guest page fault and needs to be reflected to the guest. */
15096	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
15097	TRPMResetTrap(pVCpu);
15098	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
15099	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15100	uGstErrorCode, pVmxTransient->uExitQual);
15101	}
15102	else
15103	{
15104	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15105	TRPMResetTrap(pVCpu);
15106	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
15107	hmR0VmxSetPendingXcptDF(pVCpu);
15108	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
15109	}
15110
15111	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15112	return VINF_SUCCESS;
15113	}
15114
15115	TRPMResetTrap(pVCpu);
15116	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
15117	return rc;
15118	}
15119
15120	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15121	/** @name VMX instruction handlers.
15122	* @{
15123	*/
15124	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15125	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VMX instructions VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15126	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15127
15128	/**
15129	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
15130	*/
15131	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15132	{
15133	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15134
15135	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15136	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15137	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15138	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15139	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15140	AssertRCReturn(rc, rc);
15141
15142	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15143
15144	VMXVEXITINFO ExitInfo;
15145	RT_ZERO(ExitInfo);
15146	ExitInfo.uReason = pVmxTransient->uExitReason;
15147	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15148	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15149	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15150	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15151
15152	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
15153	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15154	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15155	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15156	{
15157	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15158	rcStrict = VINF_SUCCESS;
15159	}
15160	return rcStrict;
15161	}
15162
15163
15164	/**
15165	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
15166	*/
15167	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15168	{
15169	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15170
15171	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
15172	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15173	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15174	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15175	AssertRCReturn(rc, rc);
15176
15177	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15178
15179	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
15180	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15181	{
15182	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15183	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15184	}
15185	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15186	return rcStrict;
15187	}
15188
15189
15190	/**
15191	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
15192	*/
15193	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15194	{
15195	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15196
15197	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15198	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15199	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15200	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15201	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15202	AssertRCReturn(rc, rc);
15203
15204	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15205
15206	VMXVEXITINFO ExitInfo;
15207	RT_ZERO(ExitInfo);
15208	ExitInfo.uReason = pVmxTransient->uExitReason;
15209	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15210	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15211	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15212	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15213
15214	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
15215	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15216	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15217	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15218	{
15219	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15220	rcStrict = VINF_SUCCESS;
15221	}
15222	return rcStrict;
15223	}
15224
15225
15226	/**
15227	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
15228	*/
15229	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15230	{
15231	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15232
15233	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15234	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15235	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15236	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15237	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15238	AssertRCReturn(rc, rc);
15239
15240	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15241
15242	VMXVEXITINFO ExitInfo;
15243	RT_ZERO(ExitInfo);
15244	ExitInfo.uReason = pVmxTransient->uExitReason;
15245	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15246	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15247	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15248	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15249
15250	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
15251	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15252	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15253	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15254	{
15255	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15256	rcStrict = VINF_SUCCESS;
15257	}
15258	return rcStrict;
15259	}
15260
15261
15262	/**
15263	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Unconditional VM-exit.
15264	*/
15265	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15266	{
15267	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15268
15269	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15270	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15271	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15272	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15273	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15274	AssertRCReturn(rc, rc);
15275
15276	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15277
15278	VMXVEXITINFO ExitInfo;
15279	RT_ZERO(ExitInfo);
15280	ExitInfo.uReason = pVmxTransient->uExitReason;
15281	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15282	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15283	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15284	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15285	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15286
15287	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
15288	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15289	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15290	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15291	{
15292	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15293	rcStrict = VINF_SUCCESS;
15294	}
15295	return rcStrict;
15296	}
15297
15298
15299	/**
15300	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
15301	*/
15302	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15303	{
15304	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15305
15306	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
15307	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15308	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15309	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15310	AssertRCReturn(rc, rc);
15311
15312	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15313
15314	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
15315	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15316	{
15317	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15318	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15319	}
15320	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15321	return rcStrict;
15322	}
15323
15324
15325	/**
15326	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Unconditional VM-exit.
15327	*/
15328	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15329	{
15330	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15331
15332	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15333	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15334	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15335	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15336	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15337	AssertRCReturn(rc, rc);
15338
15339	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15340
15341	VMXVEXITINFO ExitInfo;
15342	RT_ZERO(ExitInfo);
15343	ExitInfo.uReason = pVmxTransient->uExitReason;
15344	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15345	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15346	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15347	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15348	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15349
15350	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
15351	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15352	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15353	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15354	{
15355	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15356	rcStrict = VINF_SUCCESS;
15357	}
15358	return rcStrict;
15359	}
15360
15361
15362	/**
15363	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
15364	*/
15365	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15366	{
15367	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15368
15369	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15370	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
15371	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15372	AssertRCReturn(rc, rc);
15373
15374	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15375
15376	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
15377	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15378	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
15379	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15380	{
15381	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15382	rcStrict = VINF_SUCCESS;
15383	}
15384	return rcStrict;
15385	}
15386
15387
15388	/**
15389	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
15390	*/
15391	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15392	{
15393	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15394
15395	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15396	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15397	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15398	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15399	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15400	AssertRCReturn(rc, rc);
15401
15402	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15403
15404	VMXVEXITINFO ExitInfo;
15405	RT_ZERO(ExitInfo);
15406	ExitInfo.uReason = pVmxTransient->uExitReason;
15407	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15408	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15409	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15410	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15411
15412	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
15413	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15414	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15415	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15416	{
15417	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15418	rcStrict = VINF_SUCCESS;
15419	}
15420	return rcStrict;
15421	}
15422
15423	/** @} */
15424	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
15425

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 78257

Download in other formats: