HMVMXR0.cpp@ 78287

Last change on this file since 78287 was 78287, checked in by vboxsync, 6 years ago
VMM/HMVMXR0: Nested VMX: bugref:9180 Array addressing of the VMX MSR areas, easier to read and chances of missing increments are fewer. Some extra assertions.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 641.5 KB

Line
1	/* $Id: HMVMXR0.cpp 78287 2019-04-25 08:16:53Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27
28	#include <VBox/vmm/pdmapi.h>
29	#include <VBox/vmm/dbgf.h>
30	#include <VBox/vmm/iem.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#include <VBox/vmm/em.h>
35	#include <VBox/vmm/gim.h>
36	#include <VBox/vmm/apic.h>
37	#ifdef VBOX_WITH_REM
38	# include <VBox/vmm/rem.h>
39	#endif
40	#include "HMInternal.h"
41	#include <VBox/vmm/vm.h>
42	#include <VBox/vmm/hmvmxinline.h>
43	#include "HMVMXR0.h"
44	#include "dtrace/VBoxVMM.h"
45
46	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
47	#ifdef DEBUG_ramshankar
48	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
49	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
50	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
51	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
52	# define HMVMX_ALWAYS_TRAP_PF
53	# define HMVMX_ALWAYS_FLUSH_TLB
54	# define HMVMX_ALWAYS_SWAP_EFER
55	#endif
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	/** Use the function table. */
62	#define HMVMX_USE_FUNCTION_TABLE
63
64	/** Determine which tagged-TLB flush handler to use. */
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70	/** @name HMVMX_READ_XXX
71	* Flags to skip redundant reads of some common VMCS fields that are not part of
72	* the guest-CPU or VCPU state but are needed while handling VM-exits.
73	*/
74	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
75	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
76	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
77	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
78	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
79	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
80	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
81	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
82	/** @} */
83
84	/**
85	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
86	* guest using hardware-assisted VMX.
87	*
88	* This excludes state like GPRs (other than RSP) which are always are
89	* swapped and restored across the world-switch and also registers like EFER,
90	* MSR which cannot be modified by the guest without causing a VM-exit.
91	*/
92	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
93	\| CPUMCTX_EXTRN_RFLAGS \
94	\| CPUMCTX_EXTRN_RSP \
95	\| CPUMCTX_EXTRN_SREG_MASK \
96	\| CPUMCTX_EXTRN_TABLE_MASK \
97	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
98	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
99	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
100	\| CPUMCTX_EXTRN_TSC_AUX \
101	\| CPUMCTX_EXTRN_OTHER_MSRS \
102	\| CPUMCTX_EXTRN_CR0 \
103	\| CPUMCTX_EXTRN_CR3 \
104	\| CPUMCTX_EXTRN_CR4 \
105	\| CPUMCTX_EXTRN_DR7 \
106	\| CPUMCTX_EXTRN_HM_VMX_MASK)
107
108	/**
109	* Exception bitmap mask for real-mode guests (real-on-v86).
110	*
111	* We need to intercept all exceptions manually except:
112	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
113	* due to bugs in Intel CPUs.
114	* - \#PF need not be intercepted even in real-mode if we have nested paging
115	* support.
116	*/
117	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
118	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
119	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
120	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
121	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
122	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
123	\| RT_BIT(X86_XCPT_XF))
124
125	/** Maximum VM-instruction error number. */
126	#define HMVMX_INSTR_ERROR_MAX 28
127
128	/** Profiling macro. */
129	#ifdef HM_PROFILE_EXIT_DISPATCH
130	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
131	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
132	#else
133	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
134	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
135	#endif
136
137	/** Assert that preemption is disabled or covered by thread-context hooks. */
138	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
139	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
140
141	/** Assert that we haven't migrated CPUs when thread-context hooks are not
142	* used. */
143	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
144	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
145	("Illegal migration! Entered on CPU %u Current %u\n", \
146	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
147
148	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
149	* context. */
150	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
151	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
152	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
153
154	/** Helper macro for VM-exit handlers called unexpectedly. */
155	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
156	do { \
157	(a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
158	return VERR_VMX_UNEXPECTED_EXIT; \
159	} while (0)
160
161	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
162	/** Macro that does the necessary privilege checks and intercepted VM-exits for
163	* guests that attempted to execute a VMX instruction. */
164	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
165	do \
166	{ \
167	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
168	if (rcStrictTmp == VINF_SUCCESS) \
169	{ /* likely */ } \
170	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
171	{ \
172	Assert((a_pVCpu)->hm.s.Event.fPending); \
173	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
174	return VINF_SUCCESS; \
175	} \
176	else \
177	{ \
178	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
179	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
180	} \
181	} while (0)
182
183	/** Macro that decodes a memory operand for an instruction VM-exit. */
184	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
185	do \
186	{ \
187	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
188	(a_pGCPtrEffAddr)); \
189	if (rcStrictTmp == VINF_SUCCESS) \
190	{ /* likely */ } \
191	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
192	{ \
193	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
194	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
195	NOREF(uXcptTmp); \
196	return VINF_SUCCESS; \
197	} \
198	else \
199	{ \
200	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
201	return rcStrictTmp; \
202	} \
203	} while (0)
204
205	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
206
207
208	/*********************************************************************************************************************************
209	* Structures and Typedefs *
210	*********************************************************************************************************************************/
211	/**
212	* VMX transient state.
213	*
214	* A state structure for holding miscellaneous information across
215	* VMX non-root operation and restored after the transition.
216	*/
217	typedef struct VMXTRANSIENT
218	{
219	/** The host's rflags/eflags. */
220	RTCCUINTREG fEFlags;
221	#if HC_ARCH_BITS == 32
222	uint32_t u32Alignment0;
223	#endif
224	/** The guest's TPR value used for TPR shadowing. */
225	uint8_t u8GuestTpr;
226	/** Alignment. */
227	uint8_t abAlignment0[7];
228
229	/** The basic VM-exit reason. */
230	uint16_t uExitReason;
231	/** Alignment. */
232	uint16_t u16Alignment0;
233	/** The VM-exit interruption error code. */
234	uint32_t uExitIntErrorCode;
235	/** The VM-exit exit code qualification. */
236	uint64_t uExitQual;
237	/** The Guest-linear address. */
238	uint64_t uGuestLinearAddr;
239
240	/** The VM-exit interruption-information field. */
241	uint32_t uExitIntInfo;
242	/** The VM-exit instruction-length field. */
243	uint32_t cbInstr;
244	/** The VM-exit instruction-information field. */
245	VMXEXITINSTRINFO ExitInstrInfo;
246	/** Whether the VM-entry failed or not. */
247	bool fVMEntryFailed;
248	/** Whether we are currently executing a nested-guest. */
249	bool fIsNestedGuest;
250	/** Alignment. */
251	uint8_t abAlignment1[2];
252
253	/** The VM-entry interruption-information field. */
254	uint32_t uEntryIntInfo;
255	/** The VM-entry exception error code field. */
256	uint32_t uEntryXcptErrorCode;
257	/** The VM-entry instruction length field. */
258	uint32_t cbEntryInstr;
259
260	/** IDT-vectoring information field. */
261	uint32_t uIdtVectoringInfo;
262	/** IDT-vectoring error code. */
263	uint32_t uIdtVectoringErrorCode;
264
265	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
266	uint32_t fVmcsFieldsRead;
267
268	/** Whether the guest debug state was active at the time of VM-exit. */
269	bool fWasGuestDebugStateActive;
270	/** Whether the hyper debug state was active at the time of VM-exit. */
271	bool fWasHyperDebugStateActive;
272	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
273	bool fUpdatedTscOffsettingAndPreemptTimer;
274	/** Whether the VM-exit was caused by a page-fault during delivery of a
275	* contributory exception or a page-fault. */
276	bool fVectoringDoublePF;
277	/** Whether the VM-exit was caused by a page-fault during delivery of an
278	* external interrupt or NMI. */
279	bool fVectoringPF;
280	bool afAlignment0[3];
281
282	/** The VMCS info. object. */
283	PVMXVMCSINFO pVmcsInfo;
284	} VMXTRANSIENT;
285	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
286	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
287	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
288	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
289	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
290	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
291	/** Pointer to VMX transient state. */
292	typedef VMXTRANSIENT *PVMXTRANSIENT;
293
294	/**
295	* Memory operand read or write access.
296	*/
297	typedef enum VMXMEMACCESS
298	{
299	VMXMEMACCESS_READ = 0,
300	VMXMEMACCESS_WRITE = 1
301	} VMXMEMACCESS;
302
303	/**
304	* VMX VM-exit handler.
305	*
306	* @returns Strict VBox status code (i.e. informational status codes too).
307	* @param pVCpu The cross context virtual CPU structure.
308	* @param pVmxTransient The VMX-transient structure.
309	*/
310	#ifndef HMVMX_USE_FUNCTION_TABLE
311	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
312	#else
313	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
314	/** Pointer to VM-exit handler. */
315	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
316	#endif
317
318	/**
319	* VMX VM-exit handler, non-strict status code.
320	*
321	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
322	*
323	* @returns VBox status code, no informational status code returned.
324	* @param pVCpu The cross context virtual CPU structure.
325	* @param pVmxTransient The VMX-transient structure.
326	*
327	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
328	* use of that status code will be replaced with VINF_EM_SOMETHING
329	* later when switching over to IEM.
330	*/
331	#ifndef HMVMX_USE_FUNCTION_TABLE
332	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
333	#else
334	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
335	#endif
336
337
338	/*********************************************************************************************************************************
339	* Internal Functions *
340	*********************************************************************************************************************************/
341	#ifndef HMVMX_USE_FUNCTION_TABLE
342	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
343	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
344	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
345	#else
346	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
347	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
348	#endif
349	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
350	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
351	#endif
352
353	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
354	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
355	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
356	#endif
357
358	/** @name VM-exit handlers.
359	* @{
360	*/
361	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
362	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
363	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
364	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
365	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
366	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
367	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
368	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
369	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
370	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
371	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
372	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
373	static FNVMXEXITHANDLER hmR0VmxExitHlt;
374	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
375	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
376	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
377	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
378	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
379	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
380	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
381	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
382	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
383	static FNVMXEXITHANDLER hmR0VmxExitVmread;
384	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
385	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
386	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
387	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
388	#endif
389	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
390	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
391	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
392	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
393	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
394	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
395	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
396	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
397	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
398	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
399	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
400	static FNVMXEXITHANDLER hmR0VmxExitMwait;
401	static FNVMXEXITHANDLER hmR0VmxExitMtf;
402	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
403	static FNVMXEXITHANDLER hmR0VmxExitPause;
404	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
405	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
406	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
407	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
408	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
409	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
410	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
411	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
412	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
413	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
414	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
415	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
416	/** @} */
417
418	/** @name Helpers for hardware exceptions VM-exit handlers.
419	* @{
420	*/
421	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
422	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
423	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
424	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
425	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
426	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
427	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
428	/** @} */
429
430
431	/*********************************************************************************************************************************
432	* Global Variables *
433	*********************************************************************************************************************************/
434	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
435	static const uint32_t g_aVmcsCacheSegBase[] =
436	{
437	VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
438	VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
439	VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
440	VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
441	VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
442	VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
443	};
444	AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
445	#endif
446	static const uint32_t g_aVmcsSegBase[] =
447	{
448	VMX_VMCS_GUEST_ES_BASE,
449	VMX_VMCS_GUEST_CS_BASE,
450	VMX_VMCS_GUEST_SS_BASE,
451	VMX_VMCS_GUEST_DS_BASE,
452	VMX_VMCS_GUEST_FS_BASE,
453	VMX_VMCS_GUEST_GS_BASE
454	};
455	static const uint32_t g_aVmcsSegSel[] =
456	{
457	VMX_VMCS16_GUEST_ES_SEL,
458	VMX_VMCS16_GUEST_CS_SEL,
459	VMX_VMCS16_GUEST_SS_SEL,
460	VMX_VMCS16_GUEST_DS_SEL,
461	VMX_VMCS16_GUEST_FS_SEL,
462	VMX_VMCS16_GUEST_GS_SEL
463	};
464	static const uint32_t g_aVmcsSegLimit[] =
465	{
466	VMX_VMCS32_GUEST_ES_LIMIT,
467	VMX_VMCS32_GUEST_CS_LIMIT,
468	VMX_VMCS32_GUEST_SS_LIMIT,
469	VMX_VMCS32_GUEST_DS_LIMIT,
470	VMX_VMCS32_GUEST_FS_LIMIT,
471	VMX_VMCS32_GUEST_GS_LIMIT
472	};
473	static const uint32_t g_aVmcsSegAttr[] =
474	{
475	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
476	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
477	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
478	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
479	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
480	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
481	};
482	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
483	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
484	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
485	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
486
487	#ifdef HMVMX_USE_FUNCTION_TABLE
488	/**
489	* VMX_EXIT dispatch table.
490	*/
491	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
492	{
493	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
494	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
495	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
496	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
497	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
498	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
499	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
500	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
501	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
502	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
503	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
504	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
505	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
506	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
507	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
508	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
509	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
510	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
511	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
512	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
513	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
514	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
515	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
516	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
517	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
518	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
519	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
520	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
521	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
522	#else
523	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
524	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
525	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
526	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
527	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
528	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
529	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
530	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
531	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
532	#endif
533	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
534	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
535	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
536	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
537	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
538	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
539	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
540	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
541	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
542	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
543	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
544	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
545	/* 40 UNDEFINED */ hmR0VmxExitPause,
546	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
547	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
548	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
549	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
550	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
551	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitXdtrAccess,
552	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitXdtrAccess,
553	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
554	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
555	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
556	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
557	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
558	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
559	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
560	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
561	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
562	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
563	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
564	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
565	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
566	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
567	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
568	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
569	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
570	};
571	#endif /* HMVMX_USE_FUNCTION_TABLE */
572
573	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
574	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
575	{
576	/* 0 */ "(Not Used)",
577	/* 1 */ "VMCALL executed in VMX root operation.",
578	/* 2 */ "VMCLEAR with invalid physical address.",
579	/* 3 */ "VMCLEAR with VMXON pointer.",
580	/* 4 */ "VMLAUNCH with non-clear VMCS.",
581	/* 5 */ "VMRESUME with non-launched VMCS.",
582	/* 6 */ "VMRESUME after VMXOFF",
583	/* 7 */ "VM-entry with invalid control fields.",
584	/* 8 */ "VM-entry with invalid host state fields.",
585	/* 9 */ "VMPTRLD with invalid physical address.",
586	/* 10 */ "VMPTRLD with VMXON pointer.",
587	/* 11 */ "VMPTRLD with incorrect revision identifier.",
588	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
589	/* 13 */ "VMWRITE to read-only VMCS component.",
590	/* 14 */ "(Not Used)",
591	/* 15 */ "VMXON executed in VMX root operation.",
592	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
593	/* 17 */ "VM-entry with non-launched executing VMCS.",
594	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
595	/* 19 */ "VMCALL with non-clear VMCS.",
596	/* 20 */ "VMCALL with invalid VM-exit control fields.",
597	/* 21 */ "(Not Used)",
598	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
599	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
600	/* 24 */ "VMCALL with invalid SMM-monitor features.",
601	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
602	/* 26 */ "VM-entry with events blocked by MOV SS.",
603	/* 27 */ "(Not Used)",
604	/* 28 */ "Invalid operand to INVEPT/INVVPID."
605	};
606	#endif /* VBOX_STRICT */
607
608
609	/**
610	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
611	*
612	* Any bit set in this mask is owned by the host/hypervisor and would cause a
613	* VM-exit when modified by the guest.
614	*
615	* @returns The static CR0 guest/host mask.
616	* @param pVCpu The cross context virtual CPU structure.
617	*/
618	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PCVMCPU pVCpu)
619	{
620	/*
621	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
622	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
623	*/
624	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
625	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
626	* and @bugref{6944}. */
627	PVM pVM = pVCpu->CTX_SUFF(pVM);
628	return ( X86_CR0_PE
629	\| X86_CR0_NE
630	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
631	\| X86_CR0_PG
632	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
633	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
634	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
635	}
636
637
638	/**
639	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
640	*
641	* Any bit set in this mask is owned by the host/hypervisor and would cause a
642	* VM-exit when modified by the guest.
643	*
644	* @returns The static CR4 guest/host mask.
645	* @param pVCpu The cross context virtual CPU structure.
646	*/
647	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PCVMCPU pVCpu)
648	{
649	/*
650	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
651	* these bits are reserved on hardware that does not support them. Since the
652	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
653	* these bits and handle it depending on whether we expose them to the guest.
654	*/
655	PVM pVM = pVCpu->CTX_SUFF(pVM);
656	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
657	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
658	return ( X86_CR4_VMXE
659	\| X86_CR4_VME
660	\| X86_CR4_PAE
661	\| X86_CR4_PGE
662	\| X86_CR4_PSE
663	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
664	\| (fPcid ? X86_CR4_PCIDE : 0));
665	}
666
667
668	/**
669	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
670	* area.
671	*
672	* @returns @c true if it's different, @c false otherwise.
673	* @param pVmcsInfo The VMCS info. object.
674	*/
675	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
676	{
677	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
678	&& pVmcsInfo->pvGuestMsrStore);
679	}
680
681
682	/**
683	* Adds one or more exceptions to the exception bitmap and commits it to the current
684	* VMCS.
685	*
686	* @returns VBox status code.
687	* @param pVmxTransient The VMX-transient structure.
688	* @param uXcptMask The exception(s) to add.
689	*/
690	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
691	{
692	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
693	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
694	if ((uXcptBitmap & uXcptMask) != uXcptMask)
695	{
696	uXcptBitmap \|= uXcptMask;
697	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
698	AssertRCReturn(rc, rc);
699	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
700	}
701	return VINF_SUCCESS;
702	}
703
704
705	/**
706	* Adds an exception to the exception bitmap and commits it to the current VMCS.
707	*
708	* @returns VBox status code.
709	* @param pVmxTransient The VMX-transient structure.
710	* @param uXcpt The exception to add.
711	*/
712	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
713	{
714	Assert(uXcpt <= X86_XCPT_LAST);
715	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
716	}
717
718
719	/**
720	* Remove one or more exceptions from the exception bitmap and commits it to the
721	* current VMCS.
722	*
723	* This takes care of not removing the exception intercept if a nested-guest
724	* requires the exception to be intercepted.
725	*
726	* @returns VBox status code.
727	* @param pVCpu The cross context virtual CPU structure.
728	* @param pVmxTransient The VMX-transient structure.
729	* @param uXcptMask The exception(s) to remove.
730	*/
731	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
732	{
733	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
734	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
735	if (u32XcptBitmap & uXcptMask)
736	{
737	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
738	if (!pVmxTransient->fIsNestedGuest)
739	{ /* likely */ }
740	else
741	{
742	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
743	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
744	}
745	#endif
746	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
747	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
748	\| RT_BIT(X86_XCPT_DE)
749	\| RT_BIT(X86_XCPT_NM)
750	\| RT_BIT(X86_XCPT_TS)
751	\| RT_BIT(X86_XCPT_UD)
752	\| RT_BIT(X86_XCPT_NP)
753	\| RT_BIT(X86_XCPT_SS)
754	\| RT_BIT(X86_XCPT_GP)
755	\| RT_BIT(X86_XCPT_PF)
756	\| RT_BIT(X86_XCPT_MF));
757	#elif defined(HMVMX_ALWAYS_TRAP_PF)
758	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
759	#endif
760	if (uXcptMask)
761	{
762	/* Validate we are not removing any essential exception intercepts. */
763	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
764	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
765	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
766
767	/* Remove it from the exception bitmap. */
768	u32XcptBitmap &= ~uXcptMask;
769
770	/* Commit and update the cache if necessary. */
771	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
772	{
773	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
774	AssertRCReturn(rc, rc);
775	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
776	}
777	}
778	}
779	return VINF_SUCCESS;
780	}
781
782
783	/**
784	* Remove an exceptions from the exception bitmap and commits it to the current
785	* VMCS.
786	*
787	* @returns VBox status code.
788	* @param pVCpu The cross context virtual CPU structure.
789	* @param pVmxTransient The VMX-transient structure.
790	* @param uXcpt The exception to remove.
791	*/
792	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
793	{
794	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
795	}
796
797
798	/**
799	* Loads the VMCS specified by the VMCS info. object.
800	*
801	* @returns VBox status code.
802	* @param pVmcsInfo The VMCS info. object.
803	*/
804	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
805	{
806	Assert(pVmcsInfo);
807	Assert(pVmcsInfo->HCPhysVmcs);
808	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
809
810	if (pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_CLEAR)
811	{
812	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
813	if (RT_SUCCESS(rc))
814	{
815	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
816	return VINF_SUCCESS;
817	}
818	return rc;
819	}
820	return VERR_VMX_INVALID_VMCS_LAUNCH_STATE;
821	}
822
823
824	/**
825	* Clears the VMCS specified by the VMCS info. object.
826	*
827	* @returns VBox status code.
828	* @param pVmcsInfo The VMCS info. object.
829	*/
830	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
831	{
832	Assert(pVmcsInfo);
833	Assert(pVmcsInfo->HCPhysVmcs);
834	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
835
836	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
837	if (RT_SUCCESS(rc))
838	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
839	return rc;
840	}
841
842
843	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
844	/**
845	* Switches the current VMCS to the one specified.
846	*
847	* @returns VBox status code.
848	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
849	* @param pVmcsInfoTo The VMCS info. object we are switching to.
850	*
851	* @remarks Called with interrupts disabled.
852	*/
853	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
854	{
855	Assert(pVmcsInfoFrom);
856	Assert(pVmcsInfoTo);
857
858	/*
859	* Clear the VMCS we are switching out if it has not already been cleared.
860	* This will sync any CPU internal data back to the VMCS.
861	*/
862	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
863	{
864	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
865	if (RT_SUCCESS(rc))
866	{ /* likely */ }
867	else
868	return rc;
869	}
870
871	/*
872	* Clear the VMCS we are switching to if it has not already been cleared.
873	* This will initialize the VMCS launch state to "clear" required for loading it.
874	*
875	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
876	*/
877	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
878	{
879	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
880	if (RT_SUCCESS(rc))
881	{ /* likely */ }
882	else
883	return rc;
884	}
885
886	/*
887	* Finally, load the VMCS we are switching to.
888	*/
889	return hmR0VmxLoadVmcs(pVmcsInfoTo);
890	}
891	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
892
893
894	/**
895	* Updates the VM's last error record.
896	*
897	* If there was a VMX instruction error, reads the error data from the VMCS and
898	* updates VCPU's last error record as well.
899	*
900	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
901	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
902	* VERR_VMX_INVALID_VMCS_FIELD.
903	* @param rc The error code.
904	*/
905	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
906	{
907	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
908	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
909	{
910	AssertPtrReturnVoid(pVCpu);
911	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
912	}
913	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
914	}
915
916
917	#ifdef VBOX_STRICT
918	/**
919	* Reads the VM-entry interruption-information field from the VMCS into the VMX
920	* transient structure.
921	*
922	* @returns VBox status code.
923	* @param pVmxTransient The VMX-transient structure.
924	*
925	* @remarks No-long-jump zone!!!
926	*/
927	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
928	{
929	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
930	AssertRCReturn(rc, rc);
931	return VINF_SUCCESS;
932	}
933
934
935	/**
936	* Reads the VM-entry exception error code field from the VMCS into
937	* the VMX transient structure.
938	*
939	* @returns VBox status code.
940	* @param pVmxTransient The VMX-transient structure.
941	*
942	* @remarks No-long-jump zone!!!
943	*/
944	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
945	{
946	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
947	AssertRCReturn(rc, rc);
948	return VINF_SUCCESS;
949	}
950
951
952	/**
953	* Reads the VM-entry exception error code field from the VMCS into
954	* the VMX transient structure.
955	*
956	* @returns VBox status code.
957	* @param pVmxTransient The VMX-transient structure.
958	*
959	* @remarks No-long-jump zone!!!
960	*/
961	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
962	{
963	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
964	AssertRCReturn(rc, rc);
965	return VINF_SUCCESS;
966	}
967	#endif /* VBOX_STRICT */
968
969
970	/**
971	* Reads the VM-exit interruption-information field from the VMCS into the VMX
972	* transient structure.
973	*
974	* @returns VBox status code.
975	* @param pVmxTransient The VMX-transient structure.
976	*/
977	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
978	{
979	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
980	{
981	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
982	AssertRCReturn(rc,rc);
983	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
984	}
985	return VINF_SUCCESS;
986	}
987
988
989	/**
990	* Reads the VM-exit interruption error code from the VMCS into the VMX
991	* transient structure.
992	*
993	* @returns VBox status code.
994	* @param pVmxTransient The VMX-transient structure.
995	*/
996	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
997	{
998	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
999	{
1000	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1001	AssertRCReturn(rc, rc);
1002	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1003	}
1004	return VINF_SUCCESS;
1005	}
1006
1007
1008	/**
1009	* Reads the VM-exit instruction length field from the VMCS into the VMX
1010	* transient structure.
1011	*
1012	* @returns VBox status code.
1013	* @param pVmxTransient The VMX-transient structure.
1014	*/
1015	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1016	{
1017	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1018	{
1019	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1020	AssertRCReturn(rc, rc);
1021	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1022	}
1023	return VINF_SUCCESS;
1024	}
1025
1026
1027	/**
1028	* Reads the VM-exit instruction-information field from the VMCS into
1029	* the VMX transient structure.
1030	*
1031	* @returns VBox status code.
1032	* @param pVmxTransient The VMX-transient structure.
1033	*/
1034	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1035	{
1036	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1037	{
1038	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1039	AssertRCReturn(rc, rc);
1040	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1041	}
1042	return VINF_SUCCESS;
1043	}
1044
1045
1046	/**
1047	* Reads the VM-exit Qualification from the VMCS into the VMX transient structure.
1048	*
1049	* @returns VBox status code.
1050	* @param pVCpu The cross context virtual CPU structure of the
1051	* calling EMT. (Required for the VMCS cache case.)
1052	* @param pVmxTransient The VMX-transient structure.
1053	*/
1054	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1055	{
1056	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1057	{
1058	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1059	AssertRCReturn(rc, rc);
1060	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1061	}
1062	return VINF_SUCCESS;
1063	}
1064
1065
1066	/**
1067	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1068	*
1069	* @returns VBox status code.
1070	* @param pVCpu The cross context virtual CPU structure of the
1071	* calling EMT. (Required for the VMCS cache case.)
1072	* @param pVmxTransient The VMX-transient structure.
1073	*/
1074	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1075	{
1076	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1077	{
1078	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1079	AssertRCReturn(rc, rc);
1080	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1081	}
1082	return VINF_SUCCESS;
1083	}
1084
1085
1086	/**
1087	* Reads the IDT-vectoring information field from the VMCS into the VMX
1088	* transient structure.
1089	*
1090	* @returns VBox status code.
1091	* @param pVmxTransient The VMX-transient structure.
1092	*
1093	* @remarks No-long-jump zone!!!
1094	*/
1095	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1096	{
1097	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1098	{
1099	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1100	AssertRCReturn(rc, rc);
1101	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1102	}
1103	return VINF_SUCCESS;
1104	}
1105
1106
1107	/**
1108	* Reads the IDT-vectoring error code from the VMCS into the VMX
1109	* transient structure.
1110	*
1111	* @returns VBox status code.
1112	* @param pVmxTransient The VMX-transient structure.
1113	*/
1114	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1115	{
1116	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1117	{
1118	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1119	AssertRCReturn(rc, rc);
1120	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1121	}
1122	return VINF_SUCCESS;
1123	}
1124
1125
1126	/**
1127	* Enters VMX root mode operation on the current CPU.
1128	*
1129	* @returns VBox status code.
1130	* @param pVM The cross context VM structure. Can be
1131	* NULL, after a resume.
1132	* @param HCPhysCpuPage Physical address of the VMXON region.
1133	* @param pvCpuPage Pointer to the VMXON region.
1134	*/
1135	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1136	{
1137	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1138	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1139	Assert(pvCpuPage);
1140	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1141
1142	if (pVM)
1143	{
1144	/* Write the VMCS revision identifier to the VMXON region. */
1145	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1146	}
1147
1148	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1149	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1150
1151	/* Enable the VMX bit in CR4 if necessary. */
1152	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1153
1154	/* Enter VMX root mode. */
1155	int rc = VMXEnable(HCPhysCpuPage);
1156	if (RT_FAILURE(rc))
1157	{
1158	if (!(uOldCr4 & X86_CR4_VMXE))
1159	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1160
1161	if (pVM)
1162	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1163	}
1164
1165	/* Restore interrupts. */
1166	ASMSetFlags(fEFlags);
1167	return rc;
1168	}
1169
1170
1171	/**
1172	* Exits VMX root mode operation on the current CPU.
1173	*
1174	* @returns VBox status code.
1175	*/
1176	static int hmR0VmxLeaveRootMode(void)
1177	{
1178	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1179
1180	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1181	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1182
1183	/* If we're for some reason not in VMX root mode, then don't leave it. */
1184	RTCCUINTREG const uHostCR4 = ASMGetCR4();
1185
1186	int rc;
1187	if (uHostCR4 & X86_CR4_VMXE)
1188	{
1189	/* Exit VMX root mode and clear the VMX bit in CR4. */
1190	VMXDisable();
1191	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1192	rc = VINF_SUCCESS;
1193	}
1194	else
1195	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1196
1197	/* Restore interrupts. */
1198	ASMSetFlags(fEFlags);
1199	return rc;
1200	}
1201
1202
1203	/**
1204	* Allocates and maps a physically contiguous page. The allocated page is
1205	* zero'd out (used by various VT-x structures).
1206	*
1207	* @returns IPRT status code.
1208	* @param pMemObj Pointer to the ring-0 memory object.
1209	* @param ppVirt Where to store the virtual address of the
1210	* allocation.
1211	* @param pHCPhys Where to store the physical address of the
1212	* allocation.
1213	*/
1214	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1215	{
1216	AssertPtr(pMemObj);
1217	AssertPtr(ppVirt);
1218	AssertPtr(pHCPhys);
1219	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1220	if (RT_FAILURE(rc))
1221	return rc;
1222	ppVirt = RTR0MemObjAddress(pMemObj);
1223	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1224	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1225	return VINF_SUCCESS;
1226	}
1227
1228
1229	/**
1230	* Frees and unmaps an allocated, physical page.
1231	*
1232	* @param pMemObj Pointer to the ring-0 memory object.
1233	* @param ppVirt Where to re-initialize the virtual address of
1234	* allocation as 0.
1235	* @param pHCPhys Where to re-initialize the physical address of the
1236	* allocation as 0.
1237	*/
1238	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1239	{
1240	AssertPtr(pMemObj);
1241	AssertPtr(ppVirt);
1242	AssertPtr(pHCPhys);
1243	/* NULL is valid, accepted and ignored by the free function below. */
1244	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1245	*pMemObj = NIL_RTR0MEMOBJ;
1246	*ppVirt = NULL;
1247	*pHCPhys = NIL_RTHCPHYS;
1248	}
1249
1250
1251	/**
1252	* Initializes a VMCS info. object.
1253	*
1254	* @param pVmcsInfo The VMCS info. object.
1255	*/
1256	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1257	{
1258	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1259
1260	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1261	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1262	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1263	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1264	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1265	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1266	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1267	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1268	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1269	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1270	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1271	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1272	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1273	}
1274
1275
1276	/**
1277	* Frees the VT-x structures for a VMCS info. object.
1278	*
1279	* @param pVM The cross context VM structure.
1280	* @param pVmcsInfo The VMCS info. object.
1281	*/
1282	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1283	{
1284	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1285
1286	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1287	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1288
1289	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1290	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1291	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1292
1293	hmR0VmxInitVmcsInfo(pVmcsInfo);
1294	}
1295
1296
1297	/**
1298	* Allocates the VT-x structures for a VMCS info. object.
1299	*
1300	* @returns VBox status code.
1301	* @param pVCpu The cross context virtual CPU structure.
1302	* @param pVmcsInfo The VMCS info. object.
1303	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1304	*/
1305	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1306	{
1307	PVM pVM = pVCpu->CTX_SUFF(pVM);
1308
1309	/* Allocate the guest VM control structure (VMCS). */
1310	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1311	if (RT_SUCCESS(rc))
1312	{
1313	if (!fIsNstGstVmcs)
1314	{
1315	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1316	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1317	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1318	{
1319	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1320	NULL /* pR3Ptr /, NULL / pRCPtr */);
1321	}
1322	}
1323	else
1324	{
1325	Assert(pVmcsInfo->HCPhysVirtApic == NIL_RTHCPHYS);
1326	Assert(!pVmcsInfo->pbVirtApic);
1327	}
1328
1329	if (RT_SUCCESS(rc))
1330	{
1331	/*
1332	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1333	* transparent accesses of specific MSRs.
1334	*
1335	* If the condition for enabling MSR bitmaps changes here, don't forget to
1336	* update HMIsMsrBitmapActive().
1337	*
1338	* We don't share MSR bitmaps between the guest and nested-guest as we then
1339	* don't need to care about carefully restoring the guest MSR bitmap.
1340	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1341	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1342	*/
1343	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1344	{
1345	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1346	if (RT_SUCCESS(rc))
1347	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1348	}
1349
1350	if (RT_SUCCESS(rc))
1351	{
1352	/*
1353	* Allocate the VM-entry MSR-load area for the guest MSRs.
1354	*
1355	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1356	* the guest and nested-guest.
1357	*/
1358	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1359	&pVmcsInfo->HCPhysGuestMsrLoad);
1360	if (RT_SUCCESS(rc))
1361	{
1362	/*
1363	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1364	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1365	*/
1366	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1367	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1368	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1369
1370	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1371	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1372	&pVmcsInfo->HCPhysHostMsrLoad);
1373	}
1374	}
1375	}
1376	}
1377
1378	return rc;
1379	}
1380
1381
1382	/**
1383	* Free all VT-x structures for the VM.
1384	*
1385	* @returns IPRT status code.
1386	* @param pVM The cross context VM structure.
1387	*/
1388	static void hmR0VmxStructsFree(PVM pVM)
1389	{
1390	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1391	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1392	#endif
1393	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1394
1395	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1396	{
1397	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1398	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1399	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1400	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1401	if (pVM->cpum.ro.GuestFeatures.fVmx)
1402	{
1403	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1404	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1405	}
1406	#endif
1407	}
1408	}
1409
1410
1411	/**
1412	* Allocate all VT-x structures for the VM.
1413	*
1414	* @returns IPRT status code.
1415	* @param pVM The cross context VM structure.
1416	*/
1417	static int hmR0VmxStructsAlloc(PVM pVM)
1418	{
1419	/*
1420	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1421	* The VMCS size cannot be more than 4096 bytes.
1422	*
1423	* See Intel spec. Appendix A.1 "Basic VMX Information".
1424	*/
1425	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1426	if (cbVmcs <= X86_PAGE_4K_SIZE)
1427	{ /* likely */ }
1428	else
1429	{
1430	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1431	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1432	}
1433
1434	/*
1435	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1436	*/
1437	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1438	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1439	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1440	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1441	#endif
1442
1443	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1444	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1445	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1446
1447	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1448	{
1449	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1450	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1451	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1452	}
1453
1454	/*
1455	* Allocate per-VM VT-x structures.
1456	*/
1457	int rc = VINF_SUCCESS;
1458	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1459	/* Allocate crash-dump magic scratch page. */
1460	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1461	if (RT_FAILURE(rc))
1462	{
1463	hmR0VmxStructsFree(pVM);
1464	return rc;
1465	}
1466	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1467	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1468	#endif
1469
1470	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1471	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1472	{
1473	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1474	&pVM->hm.s.vmx.HCPhysApicAccess);
1475	if (RT_FAILURE(rc))
1476	{
1477	hmR0VmxStructsFree(pVM);
1478	return rc;
1479	}
1480	}
1481
1482	/*
1483	* Initialize per-VCPU VT-x structures.
1484	*/
1485	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1486	{
1487	/* Allocate the guest VMCS structures. */
1488	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1489	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
1490	if (RT_SUCCESS(rc))
1491	{
1492	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1493	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
1494	if (pVM->cpum.ro.GuestFeatures.fVmx)
1495	{
1496	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
1497	if (RT_SUCCESS(rc))
1498	{ /* likely */ }
1499	else
1500	break;
1501	}
1502	#endif
1503	}
1504	else
1505	break;
1506	}
1507
1508	if (RT_FAILURE(rc))
1509	{
1510	hmR0VmxStructsFree(pVM);
1511	return rc;
1512	}
1513
1514	return VINF_SUCCESS;
1515	}
1516
1517
1518	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1519	/**
1520	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
1521	*
1522	* @returns @c true if the MSR is intercepted, @c false otherwise.
1523	* @param pvMsrBitmap The MSR bitmap.
1524	* @param offMsr The MSR byte offset.
1525	* @param iBit The bit offset from the byte offset.
1526	*/
1527	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
1528	{
1529	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
1530	Assert(pbMsrBitmap);
1531	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
1532	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
1533	}
1534	#endif
1535
1536
1537	/**
1538	* Sets the permission bits for the specified MSR in the given MSR bitmap.
1539	*
1540	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
1541	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
1542	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
1543	* the read/write access of this MSR.
1544	*
1545	* @param pVCpu The cross context virtual CPU structure.
1546	* @param pVmcsInfo The VMCS info. object.
1547	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1548	* @param idMsr The MSR value.
1549	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
1550	* include both a read -and- a write permission!
1551	*
1552	* @sa HMGetVmxMsrPermission.
1553	* @remarks Can be called with interrupts disabled.
1554	*/
1555	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
1556	{
1557	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
1558	Assert(pbMsrBitmap);
1559	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
1560
1561	/*
1562	* MSR-bitmap Layout:
1563	* Byte index MSR range Interpreted as
1564	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1565	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1566	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1567	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1568	*
1569	* A bit corresponding to an MSR within the above range causes a VM-exit
1570	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
1571	* the MSR range, it always cause a VM-exit.
1572	*
1573	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
1574	*/
1575	uint16_t const offBitmapRead = 0;
1576	uint16_t const offBitmapWrite = 0x800;
1577	uint16_t offMsr;
1578	int32_t iBit;
1579	if (idMsr <= UINT32_C(0x00001fff))
1580	{
1581	offMsr = 0;
1582	iBit = idMsr;
1583	}
1584	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1585	{
1586	offMsr = 0x400;
1587	iBit = idMsr - UINT32_C(0xc0000000);
1588	}
1589	else
1590	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
1591
1592	/*
1593	* Set the MSR read permission.
1594	*/
1595	uint16_t const offMsrRead = offBitmapRead + offMsr;
1596	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
1597	if (fMsrpm & VMXMSRPM_ALLOW_RD)
1598	{
1599	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1600	bool const fClear = !fIsNstGstVmcs ? true
1601	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
1602	#else
1603	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1604	bool const fClear = true;
1605	#endif
1606	if (fClear)
1607	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
1608	}
1609	else
1610	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
1611
1612	/*
1613	* Set the MSR write permission.
1614	*/
1615	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
1616	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
1617	if (fMsrpm & VMXMSRPM_ALLOW_WR)
1618	{
1619	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1620	bool const fClear = !fIsNstGstVmcs ? true
1621	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
1622	#else
1623	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1624	bool const fClear = true;
1625	#endif
1626	if (fClear)
1627	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
1628	}
1629	else
1630	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
1631	}
1632
1633
1634	/**
1635	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1636	* area.
1637	*
1638	* @returns VBox status code.
1639	* @param pVCpu The cross context virtual CPU structure.
1640	* @param pVmcsInfo The VMCS info. object.
1641	* @param cMsrs The number of MSRs.
1642	*/
1643	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
1644	{
1645	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1646	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1647	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
1648	{
1649	/* Commit the MSR counts to the VMCS and update the cache. */
1650	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
1651	{
1652	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1653	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1654	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1655	AssertRCReturn(rc, rc);
1656
1657	pVmcsInfo->cEntryMsrLoad = cMsrs;
1658	pVmcsInfo->cExitMsrStore = cMsrs;
1659	pVmcsInfo->cExitMsrLoad = cMsrs;
1660	}
1661	return VINF_SUCCESS;
1662	}
1663
1664	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
1665	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1666	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1667	}
1668
1669
1670	/**
1671	* Adds a new (or updates the value of an existing) guest/host MSR
1672	* pair to be swapped during the world-switch as part of the
1673	* auto-load/store MSR area in the VMCS.
1674	*
1675	* @returns VBox status code.
1676	* @param pVCpu The cross context virtual CPU structure.
1677	* @param pVmxTransient The VMX-transient structure.
1678	* @param idMsr The MSR.
1679	* @param uGuestMsrValue Value of the guest MSR.
1680	* @param fSetReadWrite Whether to set the guest read/write access of this
1681	* MSR (thus not causing a VM-exit).
1682	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1683	* necessary.
1684	*/
1685	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
1686	bool fSetReadWrite, bool fUpdateHostMsr)
1687	{
1688	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1689	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1690	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1691	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1692	uint32_t i;
1693
1694	/* Paranoia. */
1695	Assert(pGuestMsrLoad);
1696
1697	/* Check if the MSR already exists in the VM-entry MSR-load area. */
1698	for (i = 0; i < cMsrs; i++)
1699	{
1700	if (pGuestMsrLoad[i].u32Msr == idMsr)
1701	break;
1702	}
1703
1704	bool fAdded = false;
1705	if (i == cMsrs)
1706	{
1707	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
1708	++cMsrs;
1709	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1710	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
1711
1712	/* Set the guest to read/write this MSR without causing VM-exits. */
1713	if ( fSetReadWrite
1714	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
1715	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
1716
1717	fAdded = true;
1718	}
1719
1720	/* Update the MSR value for the newly added or already existing MSR. */
1721	pGuestMsrLoad[i].u32Msr = idMsr;
1722	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
1723
1724	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
1725	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1726	{
1727	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1728	pGuestMsrStore[i].u32Msr = idMsr;
1729	pGuestMsrStore[i].u64Value = uGuestMsrValue;
1730	}
1731
1732	/* Update the corresponding slot in the host MSR area. */
1733	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1734	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
1735	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
1736	pHostMsr[i].u32Msr = idMsr;
1737
1738	/*
1739	* Only if the caller requests to update the host MSR value AND we've newly added the
1740	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
1741	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
1742	*
1743	* We do this for performance reasons since reading MSRs may be quite expensive.
1744	*/
1745	if ( fAdded
1746	&& fUpdateHostMsr)
1747	{
1748	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1749	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1750	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
1751	}
1752	return VINF_SUCCESS;
1753	}
1754
1755
1756	/**
1757	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1758	* auto-load/store MSR area in the VMCS.
1759	*
1760	* @returns VBox status code.
1761	* @param pVCpu The cross context virtual CPU structure.
1762	* @param pVmxTransient The VMX-transient structure.
1763	* @param idMsr The MSR.
1764	*/
1765	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
1766	{
1767	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1768	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1769	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1770	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1771
1772	for (uint32_t i = 0; i < cMsrs; i++)
1773	{
1774	/* Find the MSR. */
1775	if (pGuestMsrLoad[i].u32Msr == idMsr)
1776	{
1777	/*
1778	* If it's the last MSR, we only need to reduce the MSR count.
1779	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
1780	*/
1781	if (i < cMsrs - 1)
1782	{
1783	/* Remove it from the VM-entry MSR-load area. */
1784	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
1785	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
1786
1787	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
1788	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1789	{
1790	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1791	Assert(pGuestMsrStore[i].u32Msr == idMsr);
1792	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
1793	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
1794	}
1795
1796	/* Remove it from the VM-exit MSR-load area. */
1797	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1798	Assert(pHostMsr[i].u32Msr == idMsr);
1799	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
1800	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
1801	}
1802
1803	/* Reduce the count to reflect the removed MSR and bail. */
1804	--cMsrs;
1805	break;
1806	}
1807	}
1808
1809	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
1810	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
1811	{
1812	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1813	AssertRCReturn(rc, rc);
1814
1815	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1816	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1817	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
1818
1819	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
1820	return VINF_SUCCESS;
1821	}
1822
1823	return VERR_NOT_FOUND;
1824	}
1825
1826
1827	/**
1828	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
1829	*
1830	* @returns @c true if found, @c false otherwise.
1831	* @param pVmcsInfo The VMCS info. object.
1832	* @param idMsr The MSR to find.
1833	*/
1834	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
1835	{
1836	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1837	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
1838	Assert(pMsrs);
1839	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
1840	for (uint32_t i = 0; i < cMsrs; i++)
1841	{
1842	if (pMsrs[i].u32Msr == idMsr)
1843	return true;
1844	}
1845	return false;
1846	}
1847
1848
1849	/**
1850	* Updates the value of all host MSRs in the VM-exit MSR-load area.
1851	*
1852	* @param pVCpu The cross context virtual CPU structure.
1853	* @param pVmcsInfo The VMCS info. object.
1854	*
1855	* @remarks No-long-jump zone!!!
1856	*/
1857	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
1858	{
1859	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1860
1861	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1862	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
1863	Assert(pHostMsrLoad);
1864	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
1865	for (uint32_t i = 0; i < cMsrs; i++)
1866	{
1867	/*
1868	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1869	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1870	*/
1871	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
1872	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
1873	else
1874	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
1875	}
1876	}
1877
1878
1879	/**
1880	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1881	* perform lazy restoration of the host MSRs while leaving VT-x.
1882	*
1883	* @param pVCpu The cross context virtual CPU structure.
1884	*
1885	* @remarks No-long-jump zone!!!
1886	*/
1887	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1888	{
1889	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1890
1891	/*
1892	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
1893	*/
1894	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1895	{
1896	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1897	#if HC_ARCH_BITS == 64
1898	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1899	{
1900	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
1901	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
1902	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
1903	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1904	}
1905	#endif
1906	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1907	}
1908	}
1909
1910
1911	/**
1912	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1913	* lazily while leaving VT-x.
1914	*
1915	* @returns true if it does, false otherwise.
1916	* @param pVCpu The cross context virtual CPU structure.
1917	* @param idMsr The MSR to check.
1918	*/
1919	static bool hmR0VmxIsLazyGuestMsr(PCVMCPU pVCpu, uint32_t idMsr)
1920	{
1921	NOREF(pVCpu);
1922	#if HC_ARCH_BITS == 64
1923	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1924	{
1925	switch (idMsr)
1926	{
1927	case MSR_K8_LSTAR:
1928	case MSR_K6_STAR:
1929	case MSR_K8_SF_MASK:
1930	case MSR_K8_KERNEL_GS_BASE:
1931	return true;
1932	}
1933	}
1934	#else
1935	RT_NOREF(pVCpu, idMsr);
1936	#endif
1937	return false;
1938	}
1939
1940
1941	/**
1942	* Loads a set of guests MSRs to allow read/passthru to the guest.
1943	*
1944	* The name of this function is slightly confusing. This function does NOT
1945	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1946	* common prefix for functions dealing with "lazy restoration" of the shared
1947	* MSRs.
1948	*
1949	* @param pVCpu The cross context virtual CPU structure.
1950	*
1951	* @remarks No-long-jump zone!!!
1952	*/
1953	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1954	{
1955	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1956	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1957
1958	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1959	#if HC_ARCH_BITS == 64
1960	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1961	{
1962	/*
1963	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1964	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1965	* we can skip a few MSR writes.
1966	*
1967	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1968	* guest MSR values in the guest-CPU context might be different to what's currently
1969	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1970	* CPU, see @bugref{8728}.
1971	*/
1972	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1973	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1974	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
1975	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
1976	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
1977	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
1978	{
1979	#ifdef VBOX_STRICT
1980	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1981	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1982	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1983	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1984	#endif
1985	}
1986	else
1987	{
1988	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1989	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
1990	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
1991	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
1992	}
1993	}
1994	#endif
1995	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
1996	}
1997
1998
1999	/**
2000	* Performs lazy restoration of the set of host MSRs if they were previously
2001	* loaded with guest MSR values.
2002	*
2003	* @param pVCpu The cross context virtual CPU structure.
2004	*
2005	* @remarks No-long-jump zone!!!
2006	* @remarks The guest MSRs should have been saved back into the guest-CPU
2007	* context by hmR0VmxImportGuestState()!!!
2008	*/
2009	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2010	{
2011	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2012	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2013
2014	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2015	{
2016	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2017	#if HC_ARCH_BITS == 64
2018	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2019	{
2020	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2021	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2022	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2023	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2024	}
2025	#endif
2026	}
2027	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2028	}
2029
2030
2031	/**
2032	* Verifies that our cached values of the VMCS fields are all consistent with
2033	* what's actually present in the VMCS.
2034	*
2035	* @returns VBox status code.
2036	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2037	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2038	* VMCS content. HMCPU error-field is
2039	* updated, see VMX_VCI_XXX.
2040	* @param pVCpu The cross context virtual CPU structure.
2041	* @param pVmcsInfo The VMCS info. object.
2042	*/
2043	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2044	{
2045	uint32_t u32Val;
2046	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2047	AssertRCReturn(rc, rc);
2048	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2049	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32EntryCtls, u32Val),
2050	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2051	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2052
2053	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2054	AssertRCReturn(rc, rc);
2055	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2056	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ExitCtls, u32Val),
2057	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2058	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2059
2060	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2061	AssertRCReturn(rc, rc);
2062	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2063	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32PinCtls, u32Val),
2064	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2065	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2066
2067	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2068	AssertRCReturn(rc, rc);
2069	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2070	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls, u32Val),
2071	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2072	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2073
2074	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2075	{
2076	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2077	AssertRCReturn(rc, rc);
2078	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2079	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls2, u32Val),
2080	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2081	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2082	}
2083
2084	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2085	AssertRCReturn(rc, rc);
2086	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2087	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32XcptBitmap, u32Val),
2088	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2089	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2090
2091	uint64_t u64Val;
2092	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2093	AssertRCReturn(rc, rc);
2094	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2095	("Cache=%#RX64 VMCS=%#RX64\n", pVmcsInfo->u64TscOffset, u64Val),
2096	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2097	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2098
2099	return VINF_SUCCESS;
2100	}
2101
2102
2103	#ifdef VBOX_STRICT
2104	/**
2105	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2106	*
2107	* @param pVCpu The cross context virtual CPU structure.
2108	* @param pVmcsInfo The VMCS info. object.
2109	*/
2110	static void hmR0VmxCheckHostEferMsr(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2111	{
2112	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2113
2114	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2115	{
2116	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2117	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2118	uint64_t uVmcsEferMsrVmcs;
2119	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2120	AssertRC(rc);
2121
2122	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2123	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2124	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2125	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2126	}
2127	}
2128
2129
2130	/**
2131	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2132	* VMCS are correct.
2133	*
2134	* @param pVCpu The cross context virtual CPU structure.
2135	* @param pVmcsInfo The VMCS info. object.
2136	*/
2137	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2138	{
2139	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2140
2141	/* Read the various MSR-area counts from the VMCS. */
2142	uint32_t cEntryLoadMsrs;
2143	uint32_t cExitStoreMsrs;
2144	uint32_t cExitLoadMsrs;
2145	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2146	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2147	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2148
2149	/* Verify all the MSR counts are the same. */
2150	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2151	Assert(cExitStoreMsrs == cExitLoadMsrs);
2152	uint32_t const cMsrs = cExitLoadMsrs;
2153
2154	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2155	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2156
2157	/* Verify the MSR counts are within the allocated page size. */
2158	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2159
2160	/* Verify the relevant contents of the MSR areas match. */
2161	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2162	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2163	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2164	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2165	for (uint32_t i = 0; i < cMsrs; i++)
2166	{
2167	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2168	if (fSeparateExitMsrStorePage)
2169	{
2170	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2171	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2172	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2173	}
2174
2175	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2176	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2177	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2178
2179	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2180	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2181	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2182	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2183
2184	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2185	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2186	{
2187	uint32_t const fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2188	if (pGuestMsrLoad->u32Msr == MSR_K6_EFER)
2189	{
2190	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2191	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2192	}
2193	else
2194	{
2195	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2196	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2197	}
2198	}
2199
2200	/* Move to the next MSR. */
2201	pHostMsrLoad++;
2202	pGuestMsrLoad++;
2203	pGuestMsrStore++;
2204	}
2205	}
2206	#endif /* VBOX_STRICT */
2207
2208
2209	/**
2210	* Flushes the TLB using EPT.
2211	*
2212	* @returns VBox status code.
2213	* @param pVCpu The cross context virtual CPU structure of the calling
2214	* EMT. Can be NULL depending on @a enmTlbFlush.
2215	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2216	* enmTlbFlush.
2217	* @param enmTlbFlush Type of flush.
2218	*
2219	* @remarks Caller is responsible for making sure this function is called only
2220	* when NestedPaging is supported and providing @a enmTlbFlush that is
2221	* supported by the CPU.
2222	* @remarks Can be called with interrupts disabled.
2223	*/
2224	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2225	{
2226	uint64_t au64Descriptor[2];
2227	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2228	au64Descriptor[0] = 0;
2229	else
2230	{
2231	Assert(pVCpu);
2232	Assert(pVmcsInfo);
2233	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2234	}
2235	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2236
2237	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2238	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2239
2240	if ( RT_SUCCESS(rc)
2241	&& pVCpu)
2242	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2243	}
2244
2245
2246	/**
2247	* Flushes the TLB using VPID.
2248	*
2249	* @returns VBox status code.
2250	* @param pVCpu The cross context virtual CPU structure of the calling
2251	* EMT. Can be NULL depending on @a enmTlbFlush.
2252	* @param enmTlbFlush Type of flush.
2253	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2254	* on @a enmTlbFlush).
2255	*
2256	* @remarks Can be called with interrupts disabled.
2257	*/
2258	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2259	{
2260	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2261
2262	uint64_t au64Descriptor[2];
2263	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2264	{
2265	au64Descriptor[0] = 0;
2266	au64Descriptor[1] = 0;
2267	}
2268	else
2269	{
2270	AssertPtr(pVCpu);
2271	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2272	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2273	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2274	au64Descriptor[1] = GCPtr;
2275	}
2276
2277	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2278	AssertMsg(rc == VINF_SUCCESS,
2279	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2280
2281	if ( RT_SUCCESS(rc)
2282	&& pVCpu)
2283	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2284	NOREF(rc);
2285	}
2286
2287
2288	/**
2289	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2290	* otherwise there is nothing really to invalidate.
2291	*
2292	* @returns VBox status code.
2293	* @param pVCpu The cross context virtual CPU structure.
2294	* @param GCVirt Guest virtual address of the page to invalidate.
2295	*/
2296	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2297	{
2298	AssertPtr(pVCpu);
2299	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2300
2301	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2302	{
2303	/*
2304	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2305	* the EPT case. See @bugref{6043} and @bugref{6177}.
2306	*
2307	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2308	* as this function maybe called in a loop with individual addresses.
2309	*/
2310	PVM pVM = pVCpu->CTX_SUFF(pVM);
2311	if (pVM->hm.s.vmx.fVpid)
2312	{
2313	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2314
2315	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2316	/*
2317	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2318	* where executing INVVPID outside 64-bit mode does not flush translations of
2319	* 64-bit linear addresses, see @bugref{6208#c72}.
2320	*/
2321	if (RT_HI_U32(GCVirt))
2322	fVpidFlush = false;
2323	#endif
2324
2325	if (fVpidFlush)
2326	{
2327	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2328	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2329	}
2330	else
2331	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2332	}
2333	else if (pVM->hm.s.fNestedPaging)
2334	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2335	}
2336
2337	return VINF_SUCCESS;
2338	}
2339
2340
2341	/**
2342	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2343	* case where neither EPT nor VPID is supported by the CPU.
2344	*
2345	* @param pHostCpu The HM physical-CPU structure.
2346	* @param pVCpu The cross context virtual CPU structure.
2347	*
2348	* @remarks Called with interrupts disabled.
2349	*/
2350	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2351	{
2352	AssertPtr(pVCpu);
2353	AssertPtr(pHostCpu);
2354
2355	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2356
2357	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2358	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2359	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2360	pVCpu->hm.s.fForceTLBFlush = false;
2361	return;
2362	}
2363
2364
2365	/**
2366	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2367	*
2368	* @param pHostCpu The HM physical-CPU structure.
2369	* @param pVCpu The cross context virtual CPU structure.
2370	* @param pVmcsInfo The VMCS info. object.
2371	*
2372	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2373	* nomenclature. The reason is, to avoid confusion in compare statements
2374	* since the host-CPU copies are named "ASID".
2375	*
2376	* @remarks Called with interrupts disabled.
2377	*/
2378	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2379	{
2380	#ifdef VBOX_WITH_STATISTICS
2381	bool fTlbFlushed = false;
2382	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2383	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2384	if (!fTlbFlushed) \
2385	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2386	} while (0)
2387	#else
2388	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2389	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2390	#endif
2391
2392	AssertPtr(pVCpu);
2393	AssertPtr(pHostCpu);
2394	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2395
2396	PVM pVM = pVCpu->CTX_SUFF(pVM);
2397	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2398	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2399	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2400
2401	/*
2402	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2403	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2404	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2405	* cannot reuse the current ASID anymore.
2406	*/
2407	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2408	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2409	{
2410	++pHostCpu->uCurrentAsid;
2411	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2412	{
2413	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2414	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2415	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2416	}
2417
2418	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2419	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2420	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2421
2422	/*
2423	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2424	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2425	*/
2426	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2427	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2428	HMVMX_SET_TAGGED_TLB_FLUSHED();
2429	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2430	}
2431	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2432	{
2433	/*
2434	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2435	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2436	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2437	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2438	* mappings, see @bugref{6568}.
2439	*
2440	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2441	*/
2442	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2443	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2444	HMVMX_SET_TAGGED_TLB_FLUSHED();
2445	}
2446
2447	pVCpu->hm.s.fForceTLBFlush = false;
2448	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2449
2450	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
2451	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
2452	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2453	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2454	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2455	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2456	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2457	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2458	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2459
2460	/* Update VMCS with the VPID. */
2461	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2462	AssertRC(rc);
2463
2464	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2465	}
2466
2467
2468	/**
2469	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2470	*
2471	* @param pHostCpu The HM physical-CPU structure.
2472	* @param pVCpu The cross context virtual CPU structure.
2473	* @param pVmcsInfo The VMCS info. object.
2474	*
2475	* @remarks Called with interrupts disabled.
2476	*/
2477	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2478	{
2479	AssertPtr(pVCpu);
2480	AssertPtr(pHostCpu);
2481	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2482	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2483	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2484
2485	/*
2486	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2487	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2488	*/
2489	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2490	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2491	{
2492	pVCpu->hm.s.fForceTLBFlush = true;
2493	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2494	}
2495
2496	/* Check for explicit TLB flushes. */
2497	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2498	{
2499	pVCpu->hm.s.fForceTLBFlush = true;
2500	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2501	}
2502
2503	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2504	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2505
2506	if (pVCpu->hm.s.fForceTLBFlush)
2507	{
2508	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2509	pVCpu->hm.s.fForceTLBFlush = false;
2510	}
2511	}
2512
2513
2514	/**
2515	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2516	*
2517	* @param pHostCpu The HM physical-CPU structure.
2518	* @param pVCpu The cross context virtual CPU structure.
2519	*
2520	* @remarks Called with interrupts disabled.
2521	*/
2522	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2523	{
2524	AssertPtr(pVCpu);
2525	AssertPtr(pHostCpu);
2526	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2527	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2528	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2529
2530	/*
2531	* Force a TLB flush for the first world switch if the current CPU differs from the one we
2532	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2533	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2534	* cannot reuse the current ASID anymore.
2535	*/
2536	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2537	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2538	{
2539	pVCpu->hm.s.fForceTLBFlush = true;
2540	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2541	}
2542
2543	/* Check for explicit TLB flushes. */
2544	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2545	{
2546	/*
2547	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2548	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2549	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
2550	* include fExplicitFlush's too) - an obscure corner case.
2551	*/
2552	pVCpu->hm.s.fForceTLBFlush = true;
2553	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2554	}
2555
2556	PVM pVM = pVCpu->CTX_SUFF(pVM);
2557	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2558	if (pVCpu->hm.s.fForceTLBFlush)
2559	{
2560	++pHostCpu->uCurrentAsid;
2561	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2562	{
2563	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2564	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2565	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2566	}
2567
2568	pVCpu->hm.s.fForceTLBFlush = false;
2569	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2570	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2571	if (pHostCpu->fFlushAsidBeforeUse)
2572	{
2573	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2574	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2575	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2576	{
2577	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2578	pHostCpu->fFlushAsidBeforeUse = false;
2579	}
2580	else
2581	{
2582	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2583	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2584	}
2585	}
2586	}
2587
2588	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2589	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2590	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2591	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2592	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2593	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2594	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2595
2596	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2597	AssertRC(rc);
2598	}
2599
2600
2601	/**
2602	* Flushes the guest TLB entry based on CPU capabilities.
2603	*
2604	* @param pHostCpu The HM physical-CPU structure.
2605	* @param pVCpu The cross context virtual CPU structure.
2606	* @param pVmcsInfo The VMCS info. object.
2607	*
2608	* @remarks Called with interrupts disabled.
2609	*/
2610	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2611	{
2612	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2613	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2614	#endif
2615	PVM pVM = pVCpu->CTX_SUFF(pVM);
2616	switch (pVM->hm.s.vmx.enmTlbFlushType)
2617	{
2618	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
2619	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
2620	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
2621	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
2622	default:
2623	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2624	break;
2625	}
2626	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2627	}
2628
2629
2630	/**
2631	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2632	* TLB entries from the host TLB before VM-entry.
2633	*
2634	* @returns VBox status code.
2635	* @param pVM The cross context VM structure.
2636	*/
2637	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2638	{
2639	/*
2640	* Determine optimal flush type for nested paging.
2641	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
2642	* unrestricted guest execution (see hmR3InitFinalizeR0()).
2643	*/
2644	if (pVM->hm.s.fNestedPaging)
2645	{
2646	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2647	{
2648	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2649	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2650	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2651	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2652	else
2653	{
2654	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2655	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2656	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2657	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2658	}
2659
2660	/* Make sure the write-back cacheable memory type for EPT is supported. */
2661	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2662	{
2663	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2664	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2665	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2666	}
2667
2668	/* EPT requires a page-walk length of 4. */
2669	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2670	{
2671	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2672	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2673	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2674	}
2675	}
2676	else
2677	{
2678	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2679	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2680	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2681	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2682	}
2683	}
2684
2685	/*
2686	* Determine optimal flush type for VPID.
2687	*/
2688	if (pVM->hm.s.vmx.fVpid)
2689	{
2690	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2691	{
2692	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2693	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2694	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2695	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2696	else
2697	{
2698	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2699	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2700	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2701	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2702	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2703	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2704	pVM->hm.s.vmx.fVpid = false;
2705	}
2706	}
2707	else
2708	{
2709	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2710	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2711	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2712	pVM->hm.s.vmx.fVpid = false;
2713	}
2714	}
2715
2716	/*
2717	* Setup the handler for flushing tagged-TLBs.
2718	*/
2719	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2720	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2721	else if (pVM->hm.s.fNestedPaging)
2722	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2723	else if (pVM->hm.s.vmx.fVpid)
2724	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2725	else
2726	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2727	return VINF_SUCCESS;
2728	}
2729
2730
2731	/**
2732	* Sets up the virtual-APIC page address for the VMCS.
2733	*
2734	* @returns VBox status code.
2735	* @param pVCpu The cross context virtual CPU structure.
2736	* @param pVmcsInfo The VMCS info. object.
2737	*/
2738	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2739	{
2740	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2741	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
2742	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
2743	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2744	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
2745	}
2746
2747
2748	/**
2749	* Sets up the MSR-bitmap address for the VMCS.
2750	*
2751	* @returns VBox status code.
2752	* @param pVCpu The cross context virtual CPU structure.
2753	* @param pVmcsInfo The VMCS info. object.
2754	*/
2755	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2756	{
2757	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2758	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
2759	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
2760	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2761	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
2762	}
2763
2764
2765	/**
2766	* Sets up the APIC-access page address for the VMCS.
2767	*
2768	* @returns VBox status code.
2769	* @param pVCpu The cross context virtual CPU structure.
2770	*/
2771	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
2772	{
2773	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
2774	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
2775	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2776	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
2777	}
2778
2779
2780	/**
2781	* Sets up the VMCS link pointer for the VMCS.
2782	*
2783	* @returns VBox status code.
2784	* @param pVCpu The cross context virtual CPU structure.
2785	* @param pVmcsInfo The VMCS info. object.
2786	*/
2787	DECLINLINE(int) hmR0VmxSetupVmcsLinkPtr(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2788	{
2789	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2790	uint64_t const u64VmcsLinkPtr = pVmcsInfo->u64VmcsLinkPtr;
2791	Assert(u64VmcsLinkPtr == UINT64_C(0xffffffffffffffff)); /* Bits 63:0 MB1. */
2792	return VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, u64VmcsLinkPtr);
2793	}
2794
2795
2796	/**
2797	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
2798	* in the VMCS.
2799	*
2800	* @returns VBox status code.
2801	* @param pVCpu The cross context virtual CPU structure.
2802	* @param pVmcsInfo The VMCS info. object.
2803	*/
2804	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2805	{
2806	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2807
2808	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
2809	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
2810	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2811
2812	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
2813	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
2814	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
2815
2816	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
2817	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
2818	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2819
2820	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
2821	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
2822	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
2823	AssertRCReturn(rc, rc);
2824	return VINF_SUCCESS;
2825	}
2826
2827
2828	/**
2829	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
2830	*
2831	* @param pVCpu The cross context virtual CPU structure.
2832	* @param pVmcsInfo The VMCS info. object.
2833	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2834	*/
2835	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2836	{
2837	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
2838
2839	/*
2840	* The guest can access the following MSRs (read, write) without causing
2841	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
2842	*/
2843	PVM pVM = pVCpu->CTX_SUFF(pVM);
2844	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
2845	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
2846	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
2847	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2848	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
2849
2850	/*
2851	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
2852	* associated with then. We never need to intercept access (writes need to be
2853	* executed without causing a VM-exit, reads will #GP fault anyway).
2854	*
2855	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
2856	* read/write them. We swap the the guest/host MSR value using the
2857	* auto-load/store MSR area.
2858	*/
2859	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2860	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
2861	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
2862	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
2863	if (pVM->cpum.ro.GuestFeatures.fIbrs)
2864	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
2865
2866	#if HC_ARCH_BITS == 64
2867	/*
2868	* Allow full read/write access for the following MSRs (mandatory for VT-x)
2869	* required for 64-bit guests.
2870	*/
2871	if (pVM->hm.s.fAllow64BitGuests)
2872	{
2873	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
2874	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
2875	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
2876	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2877	}
2878	#endif
2879
2880	/*
2881	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
2882	*/
2883	#ifdef VBOX_STRICT
2884	Assert(pVmcsInfo->pvMsrBitmap);
2885	uint32_t const fMsrpmEfer = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
2886	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
2887	#endif
2888	}
2889
2890
2891	/**
2892	* Sets up pin-based VM-execution controls in the VMCS.
2893	*
2894	* @returns VBox status code.
2895	* @param pVCpu The cross context virtual CPU structure.
2896	* @param pVmcsInfo The VMCS info. object.
2897	*/
2898	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2899	{
2900	PVM pVM = pVCpu->CTX_SUFF(pVM);
2901	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
2902	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2903
2904	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2905	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2906
2907	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2908	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2909
2910	/* Enable the VMX-preemption timer. */
2911	if (pVM->hm.s.vmx.fUsePreemptTimer)
2912	{
2913	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2914	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
2915	}
2916
2917	#if 0
2918	/* Enable posted-interrupt processing. */
2919	if (pVM->hm.s.fPostedIntrs)
2920	{
2921	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2922	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2923	fVal \|= VMX_PIN_CTL_POSTED_INT;
2924	}
2925	#endif
2926
2927	if ((fVal & fZap) != fVal)
2928	{
2929	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2930	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
2931	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2932	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2933	}
2934
2935	/* Commit it to the VMCS and update our cache. */
2936	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2937	AssertRCReturn(rc, rc);
2938	pVmcsInfo->u32PinCtls = fVal;
2939
2940	return VINF_SUCCESS;
2941	}
2942
2943
2944	/**
2945	* Sets up secondary processor-based VM-execution controls in the VMCS.
2946	*
2947	* @returns VBox status code.
2948	* @param pVCpu The cross context virtual CPU structure.
2949	* @param pVmcsInfo The VMCS info. object.
2950	*/
2951	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2952	{
2953	PVM pVM = pVCpu->CTX_SUFF(pVM);
2954	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
2955	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2956
2957	/* WBINVD causes a VM-exit. */
2958	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2959	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
2960
2961	/* Enable EPT (aka nested-paging). */
2962	if (pVM->hm.s.fNestedPaging)
2963	fVal \|= VMX_PROC_CTLS2_EPT;
2964
2965	/* Enable the INVPCID instruction if supported by the hardware and we expose
2966	it to the guest. Without this, guest executing INVPCID would cause a #UD. */
2967	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2968	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
2969	fVal \|= VMX_PROC_CTLS2_INVPCID;
2970
2971	/* Enable VPID. */
2972	if (pVM->hm.s.vmx.fVpid)
2973	fVal \|= VMX_PROC_CTLS2_VPID;
2974
2975	/* Enable unrestricted guest execution. */
2976	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2977	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
2978
2979	#if 0
2980	if (pVM->hm.s.fVirtApicRegs)
2981	{
2982	/* Enable APIC-register virtualization. */
2983	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
2984	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
2985
2986	/* Enable virtual-interrupt delivery. */
2987	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
2988	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
2989	}
2990	#endif
2991
2992	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
2993	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2994	* done dynamically. */
2995	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
2996	{
2997	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
2998	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
2999	AssertRCReturn(rc, rc);
3000	}
3001
3002	/* Enable the RDTSCP instruction if supported by the hardware and we expose
3003	it to the guest. Without this, guest executing RDTSCP would cause a #UD. */
3004	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
3005	&& pVM->cpum.ro.GuestFeatures.fRdTscP)
3006	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3007
3008	/* Enable Pause-Loop exiting. */
3009	if ( pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT
3010	&& pVM->hm.s.vmx.cPleGapTicks
3011	&& pVM->hm.s.vmx.cPleWindowTicks)
3012	{
3013	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3014
3015	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3016	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3017	AssertRCReturn(rc, rc);
3018	}
3019
3020	if ((fVal & fZap) != fVal)
3021	{
3022	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3023	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3024	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3025	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3026	}
3027
3028	/* Commit it to the VMCS and update our cache. */
3029	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3030	AssertRCReturn(rc, rc);
3031	pVmcsInfo->u32ProcCtls2 = fVal;
3032
3033	return VINF_SUCCESS;
3034	}
3035
3036
3037	/**
3038	* Sets up processor-based VM-execution controls in the VMCS.
3039	*
3040	* @returns VBox status code.
3041	* @param pVCpu The cross context virtual CPU structure.
3042	* @param pVmcsInfo The VMCS info. object.
3043	*/
3044	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3045	{
3046	PVM pVM = pVCpu->CTX_SUFF(pVM);
3047
3048	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3049	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3050
3051	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3052	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3053	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3054	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3055	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3056	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3057	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3058
3059	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3060	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3061	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3062	{
3063	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3064	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3065	}
3066
3067	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3068	if (!pVM->hm.s.fNestedPaging)
3069	{
3070	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3071	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3072	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3073	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3074	}
3075
3076	/* Use TPR shadowing if supported by the CPU. */
3077	if ( PDMHasApic(pVM)
3078	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3079	{
3080	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3081	/* CR8 writes cause a VM-exit based on TPR threshold. */
3082	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3083	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3084	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3085	AssertRCReturn(rc, rc);
3086	}
3087	else
3088	{
3089	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3090	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3091	if (pVM->hm.s.fAllow64BitGuests)
3092	{
3093	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3094	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3095	}
3096	}
3097
3098	/* Use MSR-bitmaps if supported by the CPU. */
3099	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3100	{
3101	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3102	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3103	AssertRCReturn(rc, rc);
3104	}
3105
3106	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3107	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3108	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3109
3110	if ((fVal & fZap) != fVal)
3111	{
3112	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3113	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3114	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3115	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3116	}
3117
3118	/* Commit it to the VMCS and update our cache. */
3119	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3120	AssertRCReturn(rc, rc);
3121	pVmcsInfo->u32ProcCtls = fVal;
3122
3123	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3124	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3125	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo, false /* fIsNstGstVmcs */);
3126
3127	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3128	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3129	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3130
3131	/* Sanity check, should not really happen. */
3132	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3133	{ /* likely */ }
3134	else
3135	{
3136	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3137	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3138	}
3139
3140	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3141	return VINF_SUCCESS;
3142	}
3143
3144
3145	/**
3146	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3147	* Processor-based VM-execution) control fields in the VMCS.
3148	*
3149	* @returns VBox status code.
3150	* @param pVCpu The cross context virtual CPU structure.
3151	* @param pVmcsInfo The VMCS info. object.
3152	*/
3153	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3154	{
3155	/* Set the auto-load/store MSR area addresses in the VMCS. */
3156	int rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3157	if (RT_SUCCESS(rc))
3158	{
3159	/* Set the VMCS link pointer in the VMCS. */
3160	rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3161	if (RT_SUCCESS(rc))
3162	{
3163	/* Set the CR0/CR4 guest/host mask. */
3164	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3165	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3166	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3167	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3168	if (RT_SUCCESS(rc))
3169	{
3170	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3171	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3172	return VINF_SUCCESS;
3173	}
3174	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3175	}
3176	else
3177	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3178	}
3179	else
3180	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3181	return rc;
3182	}
3183
3184
3185	/**
3186	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3187	*
3188	* We shall setup those exception intercepts that don't change during the
3189	* lifetime of the VM here. The rest are done dynamically while loading the
3190	* guest state.
3191	*
3192	* @returns VBox status code.
3193	* @param pVCpu The cross context virtual CPU structure.
3194	* @param pVmcsInfo The VMCS info. object.
3195	*/
3196	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3197	{
3198	/*
3199	* The following exceptions are always intercepted:
3200	*
3201	* #AC - To prevent the guest from hanging the CPU.
3202	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3203	* recursive #DBs can cause a CPU hang.
3204	* #PF - To sync our shadow page tables when nested-paging is not used.
3205	*/
3206	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3207	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3208	\| RT_BIT(X86_XCPT_DB)
3209	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3210
3211	/* Commit it to the VMCS. */
3212	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3213	AssertRCReturn(rc, rc);
3214
3215	/* Update our cache of the exception bitmap. */
3216	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3217	return VINF_SUCCESS;
3218	}
3219
3220
3221	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3222	/**
3223	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3224	*
3225	* @returns VBox status code.
3226	* @param pVCpu The cross context virtual CPU structure.
3227	* @param pVmcsInfo The VMCS info. object.
3228	*/
3229	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3230	{
3231	PVM pVM = pVCpu->CTX_SUFF(pVM);
3232	int rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3233	if (RT_SUCCESS(rc))
3234	{
3235	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3236	if (RT_SUCCESS(rc))
3237	{
3238	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3239	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3240	if (RT_SUCCESS(rc))
3241	{
3242	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3243	rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3244	if (RT_SUCCESS(rc))
3245	return VINF_SUCCESS;
3246
3247	LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3248	}
3249	else
3250	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3251	}
3252	else
3253	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3254	}
3255	else
3256	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3257
3258	return rc;
3259	}
3260	#endif
3261
3262
3263	/**
3264	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3265	* VMX.
3266	*
3267	* @returns VBox status code.
3268	* @param pVCpu The cross context virtual CPU structure.
3269	* @param pVmcsInfo The VMCS info. object.
3270	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3271	*/
3272	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3273	{
3274	Assert(pVmcsInfo);
3275	Assert(pVmcsInfo->pvVmcs);
3276	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3277
3278	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3279	PVM pVM = pVCpu->CTX_SUFF(pVM);
3280	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3281	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3282
3283	LogFlowFunc(("\n"));
3284
3285	/*
3286	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3287	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3288	*/
3289	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3290	if (RT_SUCCESS(rc))
3291	{
3292	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3293	if (RT_SUCCESS(rc))
3294	{
3295	if (!fIsNstGstVmcs)
3296	{
3297	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3298	if (RT_SUCCESS(rc))
3299	{
3300	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
3301	if (RT_SUCCESS(rc))
3302	{
3303	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
3304	if (RT_SUCCESS(rc))
3305	{
3306	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
3307	if (RT_SUCCESS(rc))
3308	{ /* likely */ }
3309	else
3310	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
3311	}
3312	else
3313	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
3314	}
3315	else
3316	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
3317	}
3318	else
3319	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
3320	}
3321	else
3322	{
3323	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3324	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
3325	if (RT_SUCCESS(rc))
3326	{ /* likely */ }
3327	else
3328	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
3329	#else
3330	AssertFailed();
3331	#endif
3332	}
3333	}
3334	else
3335	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
3336	}
3337	else
3338	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
3339
3340	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
3341	if (RT_SUCCESS(rc))
3342	{
3343	rc = hmR0VmxClearVmcs(pVmcsInfo);
3344	if (RT_SUCCESS(rc))
3345	{ /* likely */ }
3346	else
3347	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
3348	}
3349
3350	/*
3351	* Update the last-error record both for failures and success, so we
3352	* can propagate the status code back to ring-3 for diagnostics.
3353	*/
3354	hmR0VmxUpdateErrorRecord(pVCpu, rc);
3355	NOREF(pszVmcs);
3356	return rc;
3357	}
3358
3359
3360	/**
3361	* Does global VT-x initialization (called during module initialization).
3362	*
3363	* @returns VBox status code.
3364	*/
3365	VMMR0DECL(int) VMXR0GlobalInit(void)
3366	{
3367	#ifdef HMVMX_USE_FUNCTION_TABLE
3368	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
3369	# ifdef VBOX_STRICT
3370	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
3371	Assert(g_apfnVMExitHandlers[i]);
3372	# endif
3373	#endif
3374	return VINF_SUCCESS;
3375	}
3376
3377
3378	/**
3379	* Does global VT-x termination (called during module termination).
3380	*/
3381	VMMR0DECL(void) VMXR0GlobalTerm()
3382	{
3383	/* Nothing to do currently. */
3384	}
3385
3386
3387	/**
3388	* Sets up and activates VT-x on the current CPU.
3389	*
3390	* @returns VBox status code.
3391	* @param pHostCpu The HM physical-CPU structure.
3392	* @param pVM The cross context VM structure. Can be
3393	* NULL after a host resume operation.
3394	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
3395	* fEnabledByHost is @c true).
3396	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
3397	* @a fEnabledByHost is @c true).
3398	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
3399	* enable VT-x on the host.
3400	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
3401	*/
3402	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
3403	PCSUPHWVIRTMSRS pHwvirtMsrs)
3404	{
3405	Assert(pHostCpu);
3406	Assert(pHwvirtMsrs);
3407	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3408
3409	/* Enable VT-x if it's not already enabled by the host. */
3410	if (!fEnabledByHost)
3411	{
3412	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
3413	if (RT_FAILURE(rc))
3414	return rc;
3415	}
3416
3417	/*
3418	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
3419	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
3420	* invalidated when flushing by VPID.
3421	*/
3422	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3423	{
3424	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
3425	pHostCpu->fFlushAsidBeforeUse = false;
3426	}
3427	else
3428	pHostCpu->fFlushAsidBeforeUse = true;
3429
3430	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
3431	++pHostCpu->cTlbFlushes;
3432
3433	return VINF_SUCCESS;
3434	}
3435
3436
3437	/**
3438	* Deactivates VT-x on the current CPU.
3439	*
3440	* @returns VBox status code.
3441	* @param pvCpuPage Pointer to the VMXON region.
3442	* @param HCPhysCpuPage Physical address of the VMXON region.
3443	*
3444	* @remarks This function should never be called when SUPR0EnableVTx() or
3445	* similar was used to enable VT-x on the host.
3446	*/
3447	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
3448	{
3449	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
3450
3451	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3452	return hmR0VmxLeaveRootMode();
3453	}
3454
3455
3456	/**
3457	* Does per-VM VT-x initialization.
3458	*
3459	* @returns VBox status code.
3460	* @param pVM The cross context VM structure.
3461	*/
3462	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
3463	{
3464	LogFlowFunc(("pVM=%p\n", pVM));
3465
3466	int rc = hmR0VmxStructsAlloc(pVM);
3467	if (RT_FAILURE(rc))
3468	{
3469	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
3470	return rc;
3471	}
3472
3473	return VINF_SUCCESS;
3474	}
3475
3476
3477	/**
3478	* Does per-VM VT-x termination.
3479	*
3480	* @returns VBox status code.
3481	* @param pVM The cross context VM structure.
3482	*/
3483	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
3484	{
3485	LogFlowFunc(("pVM=%p\n", pVM));
3486
3487	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3488	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
3489	{
3490	Assert(pVM->hm.s.vmx.pvScratch);
3491	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
3492	}
3493	#endif
3494	hmR0VmxStructsFree(pVM);
3495	return VINF_SUCCESS;
3496	}
3497
3498
3499	/**
3500	* Sets up the VM for execution using hardware-assisted VMX.
3501	* This function is only called once per-VM during initialization.
3502	*
3503	* @returns VBox status code.
3504	* @param pVM The cross context VM structure.
3505	*/
3506	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
3507	{
3508	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
3509	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3510
3511	LogFlowFunc(("pVM=%p\n", pVM));
3512
3513	/*
3514	* At least verify if VMX is enabled, since we can't check if we're in
3515	* VMX root mode or not without causing a #GP.
3516	*/
3517	RTCCUINTREG const uHostCR4 = ASMGetCR4();
3518	if (RT_LIKELY(uHostCR4 & X86_CR4_VMXE))
3519	{ /* likely */ }
3520	else
3521	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
3522
3523	/*
3524	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
3525	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
3526	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
3527	*/
3528	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3529	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
3530	\|\| !pVM->hm.s.vmx.pRealModeTSS))
3531	{
3532	LogRelFunc(("Invalid real-on-v86 state.\n"));
3533	return VERR_INTERNAL_ERROR;
3534	}
3535
3536	/* Initialize these always, see hmR3InitFinalizeR0().*/
3537	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
3538	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
3539
3540	/* Setup the tagged-TLB flush handlers. */
3541	int rc = hmR0VmxSetupTaggedTlb(pVM);
3542	if (RT_FAILURE(rc))
3543	{
3544	LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
3545	return rc;
3546	}
3547
3548	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
3549	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
3550	#if HC_ARCH_BITS == 64
3551	if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
3552	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
3553	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
3554	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
3555	#endif
3556
3557	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3558	{
3559	PVMCPU pVCpu = &pVM->aCpus[idCpu];
3560	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
3561
3562	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
3563	if (RT_SUCCESS(rc))
3564	{
3565	#if HC_ARCH_BITS == 32
3566	hmR0VmxInitVmcsReadCache(pVCpu);
3567	#endif
3568	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3569	if (pVM->cpum.ro.GuestFeatures.fVmx)
3570	{
3571	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
3572	if (RT_SUCCESS(rc))
3573	{ /* likely */ }
3574	else
3575	{
3576	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
3577	return rc;
3578	}
3579	}
3580	#endif
3581	}
3582	else
3583	{
3584	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
3585	return rc;
3586	}
3587	}
3588
3589	return VINF_SUCCESS;
3590	}
3591
3592
3593	#if HC_ARCH_BITS == 32
3594	# ifdef VBOX_ENABLE_64_BITS_GUESTS
3595	/**
3596	* Check if guest state allows safe use of 32-bit switcher again.
3597	*
3598	* Segment bases and protected mode structures must be 32-bit addressable
3599	* because the 32-bit switcher will ignore high dword when writing these VMCS
3600	* fields. See @bugref{8432} for details.
3601	*
3602	* @returns true if safe, false if must continue to use the 64-bit switcher.
3603	* @param pCtx Pointer to the guest-CPU context.
3604	*
3605	* @remarks No-long-jump zone!!!
3606	*/
3607	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
3608	{
3609	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
3610	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
3611	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3612	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3613	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
3614	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3615	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
3616	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
3617	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3618	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3619
3620	/* All good, bases are 32-bit. */
3621	return true;
3622	}
3623	# endif /* VBOX_ENABLE_64_BITS_GUESTS */
3624
3625	# ifdef VBOX_STRICT
3626	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
3627	{
3628	switch (idxField)
3629	{
3630	case VMX_VMCS_GUEST_RIP:
3631	case VMX_VMCS_GUEST_RSP:
3632	case VMX_VMCS_GUEST_SYSENTER_EIP:
3633	case VMX_VMCS_GUEST_SYSENTER_ESP:
3634	case VMX_VMCS_GUEST_GDTR_BASE:
3635	case VMX_VMCS_GUEST_IDTR_BASE:
3636	case VMX_VMCS_GUEST_CS_BASE:
3637	case VMX_VMCS_GUEST_DS_BASE:
3638	case VMX_VMCS_GUEST_ES_BASE:
3639	case VMX_VMCS_GUEST_FS_BASE:
3640	case VMX_VMCS_GUEST_GS_BASE:
3641	case VMX_VMCS_GUEST_SS_BASE:
3642	case VMX_VMCS_GUEST_LDTR_BASE:
3643	case VMX_VMCS_GUEST_TR_BASE:
3644	case VMX_VMCS_GUEST_CR3:
3645	return true;
3646	}
3647	return false;
3648	}
3649
3650	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3651	{
3652	switch (idxField)
3653	{
3654	/* Read-only fields. */
3655	case VMX_VMCS_RO_EXIT_QUALIFICATION:
3656	return true;
3657	}
3658	/* Remaining readable fields should also be writable. */
3659	return hmR0VmxIsValidWriteField(idxField);
3660	}
3661	# endif /* VBOX_STRICT */
3662
3663
3664	/**
3665	* Executes the specified handler in 64-bit mode.
3666	*
3667	* @returns VBox status code (no informational status codes).
3668	* @param pVCpu The cross context virtual CPU structure.
3669	* @param enmOp The operation to perform.
3670	* @param cParams Number of parameters.
3671	* @param paParam Array of 32-bit parameters.
3672	*/
3673	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
3674	{
3675	PVM pVM = pVCpu->CTX_SUFF(pVM);
3676	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
3677	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
3678	Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
3679	Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
3680
3681	#ifdef VBOX_STRICT
3682	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
3683	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
3684
3685	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
3686	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
3687	#endif
3688
3689	/* Disable interrupts. */
3690	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
3691
3692	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
3693	RTCPUID idHostCpu = RTMpCpuId();
3694	CPUMR0SetLApic(pVCpu, idHostCpu);
3695	#endif
3696
3697	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
3698
3699	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3700	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3701
3702	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
3703	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3704	hmR0VmxClearVmcs(pVmcsInfo);
3705
3706	/* Leave VMX root mode and disable VMX. */
3707	VMXDisable();
3708	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3709
3710	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
3711	CPUMSetHyperEIP(pVCpu, enmOp);
3712	for (int i = (int)cParams - 1; i >= 0; i--)
3713	CPUMPushHyper(pVCpu, paParam[i]);
3714
3715	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
3716
3717	/* Call the switcher. */
3718	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
3719	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
3720
3721	/* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
3722	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
3723
3724	/* Re-enter VMX root mode. */
3725	int rc2 = VMXEnable(HCPhysCpuPage);
3726	if (RT_FAILURE(rc2))
3727	{
3728	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3729	ASMSetFlags(fOldEFlags);
3730	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
3731	return rc2;
3732	}
3733
3734	/* Restore the VMCS as the current VMCS. */
3735	rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
3736	AssertRC(rc2);
3737	Assert(!(ASMGetFlags() & X86_EFL_IF));
3738	ASMSetFlags(fOldEFlags);
3739	return rc;
3740	}
3741
3742
3743	/**
3744	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
3745	* supporting 64-bit guests.
3746	*
3747	* @returns VBox status code.
3748	* @param fResume Whether to VMLAUNCH or VMRESUME.
3749	* @param pCtx Pointer to the guest-CPU context.
3750	* @param pCache Pointer to the VMCS batch cache.
3751	* @param pVM The cross context VM structure.
3752	* @param pVCpu The cross context virtual CPU structure.
3753	*/
3754	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
3755	{
3756	NOREF(fResume);
3757
3758	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3759	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3760	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3761
3762	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3763	pCache->uPos = 1;
3764	pCache->interPD = PGMGetInterPaeCR3(pVM);
3765	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
3766	#endif
3767
3768	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3769	pCache->TestIn.HCPhysCpuPage = 0;
3770	pCache->TestIn.HCPhysVmcs = 0;
3771	pCache->TestIn.pCache = 0;
3772	pCache->TestOut.HCPhysVmcs = 0;
3773	pCache->TestOut.pCache = 0;
3774	pCache->TestOut.pCtx = 0;
3775	pCache->TestOut.eflags = 0;
3776	#else
3777	NOREF(pCache);
3778	#endif
3779
3780	uint32_t aParam[10];
3781	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
3782	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
3783	aParam[2] = RT_LO_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
3784	aParam[3] = RT_HI_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
3785	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
3786	aParam[5] = 0;
3787	aParam[6] = VM_RC_ADDR(pVM, pVM);
3788	aParam[7] = 0;
3789	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
3790	aParam[9] = 0;
3791
3792	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3793	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
3794	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
3795	#endif
3796	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
3797
3798	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3799	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
3800	Assert(pCtx->dr[4] == 10);
3801	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
3802	#endif
3803
3804	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3805	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
3806	AssertMsg(pCache->TestIn.HCPhysVmcs == pVmcsInfo->HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3807	pVmcsInfo->HCPhysVmcs));
3808	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3809	pCache->TestOut.HCPhysVmcs));
3810	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
3811	pCache->TestOut.pCache));
3812	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
3813	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
3814	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
3815	pCache->TestOut.pCtx));
3816	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
3817	#endif
3818	NOREF(pCtx);
3819	return rc;
3820	}
3821	#endif
3822
3823
3824	/**
3825	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
3826	* the VMCS.
3827	*
3828	* @returns VBox status code.
3829	*/
3830	static int hmR0VmxExportHostControlRegs(void)
3831	{
3832	RTCCUINTREG uReg = ASMGetCR0();
3833	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
3834	AssertRCReturn(rc, rc);
3835
3836	uReg = ASMGetCR3();
3837	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
3838	AssertRCReturn(rc, rc);
3839
3840	uReg = ASMGetCR4();
3841	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
3842	AssertRCReturn(rc, rc);
3843	return rc;
3844	}
3845
3846
3847	/**
3848	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
3849	* the host-state area in the VMCS.
3850	*
3851	* @returns VBox status code.
3852	* @param pVCpu The cross context virtual CPU structure.
3853	*/
3854	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
3855	{
3856	#if HC_ARCH_BITS == 64
3857	/**
3858	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
3859	* requirements. See hmR0VmxExportHostSegmentRegs().
3860	*/
3861	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
3862	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
3863	{ \
3864	bool fValidSelector = true; \
3865	if ((selValue) & X86_SEL_LDT) \
3866	{ \
3867	uint32_t uAttr = ASMGetSegAttr((selValue)); \
3868	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
3869	} \
3870	if (fValidSelector) \
3871	{ \
3872	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
3873	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
3874	} \
3875	(selValue) = 0; \
3876	}
3877
3878	/*
3879	* If we've executed guest code using hardware-assisted VMX, the host-state bits
3880	* will be messed up. We should -not- save the messed up state without restoring
3881	* the original host-state, see @bugref{7240}.
3882	*
3883	* This apparently can happen (most likely the FPU changes), deal with it rather than
3884	* asserting. Was observed booting Solaris 10u10 32-bit guest.
3885	*/
3886	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
3887	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
3888	{
3889	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
3890	pVCpu->idCpu));
3891	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
3892	}
3893	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
3894	#else
3895	RT_NOREF(pVCpu);
3896	#endif
3897
3898	/*
3899	* Host DS, ES, FS and GS segment registers.
3900	*/
3901	#if HC_ARCH_BITS == 64
3902	RTSEL uSelDS = ASMGetDS();
3903	RTSEL uSelES = ASMGetES();
3904	RTSEL uSelFS = ASMGetFS();
3905	RTSEL uSelGS = ASMGetGS();
3906	#else
3907	RTSEL uSelDS = 0;
3908	RTSEL uSelES = 0;
3909	RTSEL uSelFS = 0;
3910	RTSEL uSelGS = 0;
3911	#endif
3912
3913	/*
3914	* Host CS and SS segment registers.
3915	*/
3916	RTSEL uSelCS = ASMGetCS();
3917	RTSEL uSelSS = ASMGetSS();
3918
3919	/*
3920	* Host TR segment register.
3921	*/
3922	RTSEL uSelTR = ASMGetTR();
3923
3924	#if HC_ARCH_BITS == 64
3925	/*
3926	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
3927	* gain VM-entry and restore them before we get preempted.
3928	*
3929	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
3930	*/
3931	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3932	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3933	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3934	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3935	# undef VMXLOCAL_ADJUST_HOST_SEG
3936	#endif
3937
3938	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
3939	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
3940	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
3941	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
3942	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
3943	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
3944	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
3945	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
3946	Assert(uSelCS);
3947	Assert(uSelTR);
3948
3949	/* Write these host selector fields into the host-state area in the VMCS. */
3950	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3951	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3952	#if HC_ARCH_BITS == 64
3953	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3954	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3955	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3956	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3957	#else
3958	NOREF(uSelDS);
3959	NOREF(uSelES);
3960	NOREF(uSelFS);
3961	NOREF(uSelGS);
3962	#endif
3963	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3964	AssertRCReturn(rc, rc);
3965
3966	/*
3967	* Host GDTR and IDTR.
3968	*/
3969	RTGDTR Gdtr;
3970	RTIDTR Idtr;
3971	RT_ZERO(Gdtr);
3972	RT_ZERO(Idtr);
3973	ASMGetGDTR(&Gdtr);
3974	ASMGetIDTR(&Idtr);
3975	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3976	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3977	AssertRCReturn(rc, rc);
3978
3979	#if HC_ARCH_BITS == 64
3980	/*
3981	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
3982	* them to the maximum limit (0xffff) on every VM-exit.
3983	*/
3984	if (Gdtr.cbGdt != 0xffff)
3985	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
3986
3987	/*
3988	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
3989	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
3990	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
3991	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
3992	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
3993	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
3994	* at 0xffff on hosts where we are sure it won't cause trouble.
3995	*/
3996	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
3997	if (Idtr.cbIdt < 0x0fff)
3998	# else
3999	if (Idtr.cbIdt != 0xffff)
4000	# endif
4001	{
4002	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4003	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4004	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4005	}
4006	#endif
4007
4008	/*
4009	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4010	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4011	* RPL should be too in most cases.
4012	*/
4013	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4014	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4015
4016	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4017	#if HC_ARCH_BITS == 64
4018	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4019
4020	/*
4021	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4022	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4023	* restoration if the host has something else. Task switching is not supported in 64-bit
4024	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4025	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4026	*
4027	* [1] See Intel spec. 3.5 "System Descriptor Types".
4028	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4029	*/
4030	PVM pVM = pVCpu->CTX_SUFF(pVM);
4031	Assert(pDesc->System.u4Type == 11);
4032	if ( pDesc->System.u16LimitLow != 0x67
4033	\|\| pDesc->System.u4LimitHigh)
4034	{
4035	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4036	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4037	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4038	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4039	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4040	}
4041
4042	/*
4043	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4044	*/
4045	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4046	{
4047	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4048	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4049	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4050	{
4051	/* The GDT is read-only but the writable GDT is available. */
4052	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4053	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4054	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4055	AssertRCReturn(rc, rc);
4056	}
4057	}
4058	#else
4059	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4060	#endif
4061	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4062	AssertRCReturn(rc, rc);
4063
4064	/*
4065	* Host FS base and GS base.
4066	*/
4067	#if HC_ARCH_BITS == 64
4068	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4069	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4070	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4071	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4072	AssertRCReturn(rc, rc);
4073
4074	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4075	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4076	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4077	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4078	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4079	#endif
4080	return VINF_SUCCESS;
4081	}
4082
4083
4084	/**
4085	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4086	* host-state area of the VMCS.
4087	*
4088	* These MSRs will be automatically restored on the host after every successful
4089	* VM-exit.
4090	*
4091	* @returns VBox status code.
4092	* @param pVCpu The cross context virtual CPU structure.
4093	*
4094	* @remarks No-long-jump zone!!!
4095	*/
4096	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4097	{
4098	AssertPtr(pVCpu);
4099
4100	/*
4101	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4102	* rather than swapping them on every VM-entry.
4103	*/
4104	hmR0VmxLazySaveHostMsrs(pVCpu);
4105
4106	/*
4107	* Host Sysenter MSRs.
4108	*/
4109	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4110	#if HC_ARCH_BITS == 32
4111	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4112	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4113	#else
4114	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4115	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4116	#endif
4117	AssertRCReturn(rc, rc);
4118
4119	/*
4120	* Host EFER MSR.
4121	*
4122	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4123	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4124	*/
4125	PVM pVM = pVCpu->CTX_SUFF(pVM);
4126	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4127	{
4128	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4129	AssertRCReturn(rc, rc);
4130	}
4131
4132	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4133	* hmR0VmxExportGuestEntryExitCtls(). */
4134
4135	return VINF_SUCCESS;
4136	}
4137
4138
4139	/**
4140	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4141	*
4142	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4143	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4144	*
4145	* @returns true if we need to load guest EFER, false otherwise.
4146	* @param pVCpu The cross context virtual CPU structure.
4147	*
4148	* @remarks Requires EFER, CR4.
4149	* @remarks No-long-jump zone!!!
4150	*/
4151	static bool hmR0VmxShouldSwapEferMsr(PCVMCPU pVCpu)
4152	{
4153	#ifdef HMVMX_ALWAYS_SWAP_EFER
4154	RT_NOREF(pVCpu);
4155	return true;
4156	#else
4157	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4158	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4159	/* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4160	if (CPUMIsGuestInLongModeEx(pCtx))
4161	return false;
4162	#endif
4163
4164	PVM pVM = pVCpu->CTX_SUFF(pVM);
4165	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4166	uint64_t const u64GuestEfer = pCtx->msrEFER;
4167
4168	/*
4169	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4170	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4171	*/
4172	if ( CPUMIsGuestInLongModeEx(pCtx)
4173	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4174	return true;
4175
4176	/*
4177	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4178	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4179	*
4180	* See Intel spec. 4.5 "IA-32e Paging".
4181	* See Intel spec. 4.1.1 "Three Paging Modes".
4182	*
4183	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4184	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4185	*/
4186	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4187	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4188	if ( (pCtx->cr4 & X86_CR4_PAE)
4189	&& (pCtx->cr0 & X86_CR0_PG)
4190	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4191	{
4192	/* Assert that host is NX capable. */
4193	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4194	return true;
4195	}
4196
4197	return false;
4198	#endif
4199	}
4200
4201	/**
4202	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4203	* VMCS.
4204	*
4205	* This is typically required when the guest changes paging mode.
4206	*
4207	* @returns VBox status code.
4208	* @param pVCpu The cross context virtual CPU structure.
4209	* @param pVmxTransient The VMX-transient structure.
4210	*
4211	* @remarks Requires EFER.
4212	* @remarks No-long-jump zone!!!
4213	*/
4214	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4215	{
4216	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4217	{
4218	PVM pVM = pVCpu->CTX_SUFF(pVM);
4219	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4220
4221	/*
4222	* VM-entry controls.
4223	*/
4224	{
4225	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4226	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4227
4228	/*
4229	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4230	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4231	*
4232	* For nested-guests, this is a mandatory VM-entry control. It's also
4233	* required because we do not want to leak host bits to the nested-guest.
4234	*/
4235	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4236
4237	/*
4238	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4239	*
4240	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4241	* required to get the nested-guest working with hardware-assisted VMX execution.
4242	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested-hypervisor
4243	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4244	* here rather than while merging the guest VMCS controls.
4245	*/
4246	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4247	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4248	else
4249	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4250
4251	/*
4252	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4253	*
4254	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4255	* regardless of whether the nested-guest VMCS specifies it because we are free to
4256	* load whatever MSRs we require and we do not need to modify the guest visible copy
4257	* of the VM-entry MSR load area.
4258	*/
4259	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4260	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4261	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4262	else
4263	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4264
4265	/*
4266	* The following should -not- be set (since we're not in SMM mode):
4267	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4268	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4269	*/
4270
4271	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4272	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4273
4274	if ((fVal & fZap) == fVal)
4275	{ /* likely */ }
4276	else
4277	{
4278	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4279	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4280	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4281	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4282	}
4283
4284	/* Commit it to the VMCS. */
4285	if (pVmcsInfo->u32EntryCtls != fVal)
4286	{
4287	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4288	AssertRCReturn(rc, rc);
4289	pVmcsInfo->u32EntryCtls = fVal;
4290	}
4291	}
4292
4293	/*
4294	* VM-exit controls.
4295	*/
4296	{
4297	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4298	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4299
4300	/*
4301	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4302	* supported the 1-setting of this bit.
4303	*
4304	* For nested-guests, we set the "save debug controls" as the converse
4305	* "load debug controls" is mandatory for nested-guests anyway.
4306	*/
4307	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4308
4309	/*
4310	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4311	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4312	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4313	* hmR0VmxExportHostMsrs().
4314	*
4315	* For nested-guests, we always set this bit as we do not support 32-bit
4316	* hosts.
4317	*/
4318	#if HC_ARCH_BITS == 64
4319	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4320	#else
4321	Assert(!pVmxTransient->fIsNestedGuest);
4322	Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
4323	\|\| pVmcsInfo->pfnStartVM == VMXR0StartVM32);
4324	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
4325	if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
4326	{
4327	/* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
4328	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4329	}
4330	else
4331	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
4332	#endif
4333
4334	/*
4335	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4336	*
4337	* For nested-guests, we should use the "save IA32_EFER" control if we also
4338	* used the "load IA32_EFER" control while exporting VM-entry controls.
4339	*/
4340	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4341	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4342	{
4343	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4344	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4345	}
4346
4347	/*
4348	* Enable saving of the VMX-preemption timer value on VM-exit.
4349	* For nested-guests, currently not exposed/used.
4350	*/
4351	if ( pVM->hm.s.vmx.fUsePreemptTimer
4352	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4353	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4354
4355	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4356	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4357
4358	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4359	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4360	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4361
4362	if ((fVal & fZap) == fVal)
4363	{ /* likely */ }
4364	else
4365	{
4366	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4367	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4368	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4369	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4370	}
4371
4372	/* Commit it to the VMCS. */
4373	if (pVmcsInfo->u32ExitCtls != fVal)
4374	{
4375	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4376	AssertRCReturn(rc, rc);
4377	pVmcsInfo->u32ExitCtls = fVal;
4378	}
4379	}
4380
4381	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4382	}
4383	return VINF_SUCCESS;
4384	}
4385
4386
4387	/**
4388	* Sets the TPR threshold in the VMCS.
4389	*
4390	* @returns VBox status code.
4391	* @param pVCpu The cross context virtual CPU structure.
4392	* @param pVmcsInfo The VMCS info. object.
4393	* @param u32TprThreshold The TPR threshold (task-priority class only).
4394	*/
4395	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4396	{
4397	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4398	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4399	RT_NOREF2(pVCpu, pVmcsInfo);
4400	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4401	}
4402
4403
4404	/**
4405	* Exports the guest APIC TPR state into the VMCS.
4406	*
4407	* @returns VBox status code.
4408	* @param pVCpu The cross context virtual CPU structure.
4409	* @param pVmxTransient The VMX-transient structure.
4410	*
4411	* @remarks No-long-jump zone!!!
4412	*/
4413	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4414	{
4415	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4416	{
4417	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4418
4419	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4420	if (!pVmxTransient->fIsNestedGuest)
4421	{
4422	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4423	&& APICIsEnabled(pVCpu))
4424	{
4425	/*
4426	* Setup TPR shadowing.
4427	*/
4428	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4429	{
4430	bool fPendingIntr = false;
4431	uint8_t u8Tpr = 0;
4432	uint8_t u8PendingIntr = 0;
4433	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4434	AssertRCReturn(rc, rc);
4435
4436	/*
4437	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4438	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4439	* priority of the pending interrupt so we can deliver the interrupt. If there
4440	* are no interrupts pending, set threshold to 0 to not cause any
4441	* TPR-below-threshold VM-exits.
4442	*/
4443	Assert(pVmcsInfo->pbVirtApic);
4444	pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
4445	uint32_t u32TprThreshold = 0;
4446	if (fPendingIntr)
4447	{
4448	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4449	(which is the Task-Priority Class). */
4450	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4451	const uint8_t u8TprPriority = u8Tpr >> 4;
4452	if (u8PendingPriority <= u8TprPriority)
4453	u32TprThreshold = u8PendingPriority;
4454	}
4455
4456	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
4457	AssertRCReturn(rc, rc);
4458	}
4459	}
4460	}
4461	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4462	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4463	}
4464	return VINF_SUCCESS;
4465	}
4466
4467
4468	/**
4469	* Gets the guest interruptibility-state.
4470	*
4471	* @returns Guest's interruptibility-state.
4472	* @param pVCpu The cross context virtual CPU structure.
4473	* @param pVmcsInfo The VMCS info. object.
4474	*
4475	* @remarks No-long-jump zone!!!
4476	*/
4477	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
4478	{
4479	/*
4480	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4481	*/
4482	uint32_t fIntrState = 0;
4483	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4484	{
4485	/* If inhibition is active, RIP and RFLAGS should've been updated
4486	(i.e. read previously from the VMCS or from ring-3). */
4487	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4488	#ifdef VBOX_STRICT
4489	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
4490	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4491	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
4492	#endif
4493	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
4494	{
4495	if (pCtx->eflags.Bits.u1IF)
4496	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4497	else
4498	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
4499	}
4500	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4501	{
4502	/*
4503	* We can clear the inhibit force flag as even if we go back to the recompiler
4504	* without executing guest code in VT-x, the flag's condition to be cleared is
4505	* met and thus the cleared state is correct.
4506	*/
4507	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4508	}
4509	}
4510
4511	/*
4512	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
4513	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
4514	* setting this would block host-NMIs and IRET will not clear the blocking.
4515	*
4516	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
4517	*
4518	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
4519	*/
4520	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
4521	&& CPUMIsGuestNmiBlocking(pVCpu))
4522	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
4523
4524	return fIntrState;
4525	}
4526
4527
4528	/**
4529	* Exports the exception intercepts required for guest execution in the VMCS.
4530	*
4531	* @returns VBox status code.
4532	* @param pVCpu The cross context virtual CPU structure.
4533	* @param pVmxTransient The VMX-transient structure.
4534	*
4535	* @remarks No-long-jump zone!!!
4536	*/
4537	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4538	{
4539	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
4540	{
4541	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
4542	if ( !pVmxTransient->fIsNestedGuest
4543	&& pVCpu->hm.s.fGIMTrapXcptUD)
4544	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
4545	else
4546	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
4547
4548	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
4549	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
4550	}
4551	return VINF_SUCCESS;
4552	}
4553
4554
4555	/**
4556	* Exports the guest's RIP into the guest-state area in the VMCS.
4557	*
4558	* @returns VBox status code.
4559	* @param pVCpu The cross context virtual CPU structure.
4560	*
4561	* @remarks No-long-jump zone!!!
4562	*/
4563	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
4564	{
4565	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4566	{
4567	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
4568
4569	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
4570	AssertRCReturn(rc, rc);
4571
4572	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
4573	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
4574	}
4575	return VINF_SUCCESS;
4576	}
4577
4578
4579	/**
4580	* Exports the guest's RSP into the guest-state area in the VMCS.
4581	*
4582	* @returns VBox status code.
4583	* @param pVCpu The cross context virtual CPU structure.
4584	*
4585	* @remarks No-long-jump zone!!!
4586	*/
4587	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
4588	{
4589	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
4590	{
4591	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
4592
4593	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
4594	AssertRCReturn(rc, rc);
4595
4596	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
4597	}
4598	return VINF_SUCCESS;
4599	}
4600
4601
4602	/**
4603	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
4604	*
4605	* @returns VBox status code.
4606	* @param pVCpu The cross context virtual CPU structure.
4607	* @param pVmxTransient The VMX-transient structure.
4608	*
4609	* @remarks No-long-jump zone!!!
4610	*/
4611	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4612	{
4613	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
4614	{
4615	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
4616
4617	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
4618	Let us assert it as such and use 32-bit VMWRITE. */
4619	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
4620	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
4621	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
4622	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
4623
4624	/*
4625	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
4626	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
4627	* can run the real-mode guest code under Virtual 8086 mode.
4628	*/
4629	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4630	if (pVmcsInfo->RealMode.fRealOnV86Active)
4631	{
4632	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4633	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4634	Assert(!pVmxTransient->fIsNestedGuest);
4635	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
4636	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
4637	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
4638	}
4639
4640	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
4641	AssertRCReturn(rc, rc);
4642
4643	/*
4644	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
4645	*
4646	* We must avoid setting any automatic debug exceptions delivery when single-stepping
4647	* through the hypervisor debugger using EFLAGS.TF.
4648	*/
4649	if ( !pVmxTransient->fIsNestedGuest
4650	&& !pVCpu->hm.s.fSingleInstruction
4651	&& fEFlags.Bits.u1TF)
4652	{
4653	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
4654	* premature trips to ring-3 esp since IEM does not yet handle it. */
4655	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
4656	AssertRCReturn(rc, rc);
4657	}
4658	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
4659	* nested-guest VMCS. */
4660
4661	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
4662	Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
4663	}
4664	return VINF_SUCCESS;
4665	}
4666
4667
4668	/**
4669	* Exports the guest CR0 control register into the guest-state area in the VMCS.
4670	*
4671	* The guest FPU state is always pre-loaded hence we don't need to bother about
4672	* sharing FPU related CR0 bits between the guest and host.
4673	*
4674	* @returns VBox status code.
4675	* @param pVCpu The cross context virtual CPU structure.
4676	* @param pVmxTransient The VMX-transient structure.
4677	*
4678	* @remarks No-long-jump zone!!!
4679	*/
4680	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4681	{
4682	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
4683	{
4684	PVM pVM = pVCpu->CTX_SUFF(pVM);
4685	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4686
4687	/*
4688	* Figure out fixed CR0 bits in VMX operation.
4689	*/
4690	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4691	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4692	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4693	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
4694	else
4695	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
4696
4697	if (!pVmxTransient->fIsNestedGuest)
4698	{
4699	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4700	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4701	uint64_t const u64ShadowCr0 = u64GuestCr0;
4702	Assert(!RT_HI_U32(u64GuestCr0));
4703
4704	/*
4705	* Setup VT-x's view of the guest CR0.
4706	*/
4707	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
4708	if (pVM->hm.s.fNestedPaging)
4709	{
4710	if (CPUMIsGuestPagingEnabled(pVCpu))
4711	{
4712	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
4713	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
4714	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
4715	}
4716	else
4717	{
4718	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
4719	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
4720	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
4721	}
4722
4723	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
4724	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4725	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
4726	}
4727	else
4728	{
4729	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
4730	u64GuestCr0 \|= X86_CR0_WP;
4731	}
4732
4733	/*
4734	* Guest FPU bits.
4735	*
4736	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
4737	* using CR0.TS.
4738	*
4739	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
4740	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
4741	*/
4742	u64GuestCr0 \|= X86_CR0_NE;
4743
4744	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
4745	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
4746
4747	/*
4748	* Update exception intercepts.
4749	*/
4750	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
4751	if (pVmcsInfo->RealMode.fRealOnV86Active)
4752	{
4753	Assert(PDMVmmDevHeapIsEnabled(pVM));
4754	Assert(pVM->hm.s.vmx.pRealModeTSS);
4755	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
4756	}
4757	else
4758	{
4759	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
4760	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
4761	if (fInterceptMF)
4762	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
4763	}
4764
4765	/* Additional intercepts for debugging, define these yourself explicitly. */
4766	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4767	uXcptBitmap \|= 0
4768	\| RT_BIT(X86_XCPT_BP)
4769	\| RT_BIT(X86_XCPT_DE)
4770	\| RT_BIT(X86_XCPT_NM)
4771	\| RT_BIT(X86_XCPT_TS)
4772	\| RT_BIT(X86_XCPT_UD)
4773	\| RT_BIT(X86_XCPT_NP)
4774	\| RT_BIT(X86_XCPT_SS)
4775	\| RT_BIT(X86_XCPT_GP)
4776	\| RT_BIT(X86_XCPT_PF)
4777	\| RT_BIT(X86_XCPT_MF)
4778	;
4779	#elif defined(HMVMX_ALWAYS_TRAP_PF)
4780	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
4781	#endif
4782	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
4783	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
4784	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
4785
4786	/* Apply the fixed CR0 bits and enable caching. */
4787	u64GuestCr0 \|= fSetCr0;
4788	u64GuestCr0 &= fZapCr0;
4789	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4790
4791	/* Commit the CR0 and related fields to the guest VMCS. */
4792	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo Fix to 64-bit when we drop 32-bit. */
4793	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4794	if (uProcCtls != pVmcsInfo->u32ProcCtls)
4795	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4796	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
4797	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
4798	AssertRCReturn(rc, rc);
4799
4800	/* Update our caches. */
4801	pVmcsInfo->u32ProcCtls = uProcCtls;
4802	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
4803
4804	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4805	}
4806	else
4807	{
4808	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
4809	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4810	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4811	uint64_t const u64ShadowCr0 = pVmcsNstGst->u64Cr0ReadShadow.u;
4812	Assert(!RT_HI_U32(u64GuestCr0));
4813	Assert(u64GuestCr0 & X86_CR0_NE);
4814
4815	/* Apply the fixed CR0 bits and enable caching. */
4816	u64GuestCr0 \|= fSetCr0;
4817	u64GuestCr0 &= fZapCr0;
4818	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4819
4820	/* Commit the CR0 and CR0 read shadow to the nested-guest VMCS. */
4821	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo NSTVMX: Fix to 64-bit when we drop 32-bit. */
4822	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4823	AssertRCReturn(rc, rc);
4824
4825	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4826	}
4827
4828	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
4829	}
4830
4831	return VINF_SUCCESS;
4832	}
4833
4834
4835	/**
4836	* Exports the guest control registers (CR3, CR4) into the guest-state area
4837	* in the VMCS.
4838	*
4839	* @returns VBox strict status code.
4840	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
4841	* without unrestricted guest access and the VMMDev is not presently
4842	* mapped (e.g. EFI32).
4843	*
4844	* @param pVCpu The cross context virtual CPU structure.
4845	* @param pVmxTransient The VMX-transient structure.
4846	*
4847	* @remarks No-long-jump zone!!!
4848	*/
4849	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4850	{
4851	int rc = VINF_SUCCESS;
4852	PVM pVM = pVCpu->CTX_SUFF(pVM);
4853
4854	/*
4855	* Guest CR2.
4856	* It's always loaded in the assembler code. Nothing to do here.
4857	*/
4858
4859	/*
4860	* Guest CR3.
4861	*/
4862	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
4863	{
4864	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
4865
4866	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
4867	if (pVM->hm.s.fNestedPaging)
4868	{
4869	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4870	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
4871
4872	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
4873	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
4874	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
4875	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
4876
4877	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
4878	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
4879	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
4880
4881	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
4882	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
4883	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
4884	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
4885	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
4886	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
4887	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
4888
4889	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
4890	AssertRCReturn(rc, rc);
4891
4892	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4893	if ( pVM->hm.s.vmx.fUnrestrictedGuest
4894	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
4895	{
4896	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
4897	if (CPUMIsGuestInPAEModeEx(pCtx))
4898	{
4899	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
4900	AssertRCReturn(rc, rc);
4901	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
4902	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
4903	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
4904	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
4905	AssertRCReturn(rc, rc);
4906	}
4907
4908	/*
4909	* The guest's view of its CR3 is unblemished with nested paging when the
4910	* guest is using paging or we have unrestricted guest execution to handle
4911	* the guest when it's not using paging.
4912	*/
4913	GCPhysGuestCR3 = pCtx->cr3;
4914	}
4915	else
4916	{
4917	/*
4918	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
4919	* thinks it accesses physical memory directly, we use our identity-mapped
4920	* page table to map guest-linear to guest-physical addresses. EPT takes care
4921	* of translating it to host-physical addresses.
4922	*/
4923	RTGCPHYS GCPhys;
4924	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4925
4926	/* We obtain it here every time as the guest could have relocated this PCI region. */
4927	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4928	if (RT_SUCCESS(rc))
4929	{ /* likely */ }
4930	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4931	{
4932	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
4933	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4934	}
4935	else
4936	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4937
4938	GCPhysGuestCR3 = GCPhys;
4939	}
4940
4941	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
4942	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4943	AssertRCReturn(rc, rc);
4944	}
4945	else
4946	{
4947	/* Non-nested paging case, just use the hypervisor's CR3. */
4948	RTHCPHYS const HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4949
4950	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
4951	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4952	AssertRCReturn(rc, rc);
4953	}
4954
4955	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
4956	}
4957
4958	/*
4959	* Guest CR4.
4960	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4961	*/
4962	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
4963	{
4964	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4965	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4966	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
4967
4968	/*
4969	* Figure out fixed CR4 bits in VMX operation.
4970	*/
4971	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
4972	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
4973
4974	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
4975	uint64_t u64GuestCr4 = pCtx->cr4;
4976	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest ? pCtx->cr4 : pVmcsNstGst->u64Cr4ReadShadow.u;
4977	Assert(!RT_HI_U32(u64GuestCr4));
4978
4979	/*
4980	* Setup VT-x's view of the guest CR4.
4981	*
4982	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
4983	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
4984	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4985	*
4986	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4987	*/
4988	if (pVmcsInfo->RealMode.fRealOnV86Active)
4989	{
4990	Assert(pVM->hm.s.vmx.pRealModeTSS);
4991	Assert(PDMVmmDevHeapIsEnabled(pVM));
4992	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
4993	}
4994
4995	if (pVM->hm.s.fNestedPaging)
4996	{
4997	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
4998	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
4999	{
5000	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5001	u64GuestCr4 \|= X86_CR4_PSE;
5002	/* Our identity mapping is a 32-bit page directory. */
5003	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5004	}
5005	/* else use guest CR4.*/
5006	}
5007	else
5008	{
5009	Assert(!pVmxTransient->fIsNestedGuest);
5010
5011	/*
5012	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5013	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5014	*/
5015	switch (pVCpu->hm.s.enmShadowMode)
5016	{
5017	case PGMMODE_REAL: /* Real-mode. */
5018	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5019	case PGMMODE_32_BIT: /* 32-bit paging. */
5020	{
5021	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5022	break;
5023	}
5024
5025	case PGMMODE_PAE: /* PAE paging. */
5026	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5027	{
5028	u64GuestCr4 \|= X86_CR4_PAE;
5029	break;
5030	}
5031
5032	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5033	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5034	#ifdef VBOX_ENABLE_64_BITS_GUESTS
5035	break;
5036	#endif
5037	default:
5038	AssertFailed();
5039	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5040	}
5041	}
5042
5043	/* Apply the fixed CR4 bits (mainly CR4.VMXE). */
5044	u64GuestCr4 \|= fSetCr4;
5045	u64GuestCr4 &= fZapCr4;
5046
5047	/* Commit the CR4 and CR4 read shadow to the guest VMCS. */
5048	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u64GuestCr4); /** @todo Fix to 64-bit when we drop 32-bit. */
5049	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
5050	AssertRCReturn(rc, rc);
5051
5052	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5053	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5054
5055	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5056
5057	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5058	}
5059	return rc;
5060	}
5061
5062
5063	/**
5064	* Exports the guest debug registers into the guest-state area in the VMCS.
5065	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5066	*
5067	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5068	*
5069	* @returns VBox status code.
5070	* @param pVCpu The cross context virtual CPU structure.
5071	* @param pVmxTransient The VMX-transient structure.
5072	*
5073	* @remarks No-long-jump zone!!!
5074	*/
5075	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5076	{
5077	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5078
5079	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5080	* stepping. */
5081	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5082	if (pVmxTransient->fIsNestedGuest)
5083	{
5084	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5085	AssertRCReturn(rc, rc);
5086	return VINF_SUCCESS;
5087	}
5088
5089	#ifdef VBOX_STRICT
5090	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5091	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5092	{
5093	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5094	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5095	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5096	}
5097	#endif
5098
5099	bool fSteppingDB = false;
5100	bool fInterceptMovDRx = false;
5101	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5102	if (pVCpu->hm.s.fSingleInstruction)
5103	{
5104	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5105	PVM pVM = pVCpu->CTX_SUFF(pVM);
5106	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5107	{
5108	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5109	Assert(fSteppingDB == false);
5110	}
5111	else
5112	{
5113	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5114	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5115	pVCpu->hm.s.fClearTrapFlag = true;
5116	fSteppingDB = true;
5117	}
5118	}
5119
5120	uint32_t u32GuestDr7;
5121	if ( fSteppingDB
5122	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5123	{
5124	/*
5125	* Use the combined guest and host DRx values found in the hypervisor register set
5126	* because the hypervisor debugger has breakpoints active or someone is single stepping
5127	* on the host side without a monitor trap flag.
5128	*
5129	* Note! DBGF expects a clean DR6 state before executing guest code.
5130	*/
5131	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5132	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5133	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
5134	{
5135	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5136	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
5137	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
5138	}
5139	else
5140	#endif
5141	if (!CPUMIsHyperDebugStateActive(pVCpu))
5142	{
5143	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5144	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5145	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5146	}
5147
5148	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5149	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
5150	pVCpu->hm.s.fUsingHyperDR7 = true;
5151	fInterceptMovDRx = true;
5152	}
5153	else
5154	{
5155	/*
5156	* If the guest has enabled debug registers, we need to load them prior to
5157	* executing guest code so they'll trigger at the right time.
5158	*/
5159	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5160	{
5161	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5162	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5163	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
5164	{
5165	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5166	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
5167	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
5168	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5169	}
5170	else
5171	#endif
5172	if (!CPUMIsGuestDebugStateActive(pVCpu))
5173	{
5174	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5175	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5176	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5177	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5178	}
5179	Assert(!fInterceptMovDRx);
5180	}
5181	/*
5182	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5183	* must intercept #DB in order to maintain a correct DR6 guest value, and
5184	* because we need to intercept it to prevent nested #DBs from hanging the
5185	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5186	*/
5187	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5188	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
5189	&& !CPUMIsGuestDebugStateActive(pVCpu))
5190	#else
5191	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5192	#endif
5193	{
5194	fInterceptMovDRx = true;
5195	}
5196
5197	/* Update DR7 with the actual guest value. */
5198	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5199	pVCpu->hm.s.fUsingHyperDR7 = false;
5200	}
5201
5202	if (fInterceptMovDRx)
5203	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5204	else
5205	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5206
5207	/*
5208	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5209	* monitor-trap flag and update our cache.
5210	*/
5211	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5212	{
5213	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5214	AssertRCReturn(rc2, rc2);
5215	pVmcsInfo->u32ProcCtls = uProcCtls;
5216	}
5217
5218	/*
5219	* Update guest DR7.
5220	*/
5221	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
5222	AssertRCReturn(rc, rc);
5223
5224	/*
5225	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
5226	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
5227	*
5228	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5229	*/
5230	if (fSteppingDB)
5231	{
5232	Assert(pVCpu->hm.s.fSingleInstruction);
5233	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
5234
5235	uint32_t fIntrState = 0;
5236	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
5237	AssertRCReturn(rc, rc);
5238
5239	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5240	{
5241	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5242	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5243	AssertRCReturn(rc, rc);
5244	}
5245	}
5246
5247	return VINF_SUCCESS;
5248	}
5249
5250
5251	#ifdef VBOX_STRICT
5252	/**
5253	* Strict function to validate segment registers.
5254	*
5255	* @param pVCpu The cross context virtual CPU structure.
5256	* @param pVmcsInfo The VMCS info. object.
5257	*
5258	* @remarks Will import guest CR0 on strict builds during validation of
5259	* segments.
5260	*/
5261	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
5262	{
5263	/*
5264	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5265	*
5266	* The reason we check for attribute value 0 in this function and not just the unusable bit is
5267	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
5268	* unusable bit and doesn't change the guest-context value.
5269	*/
5270	PVM pVM = pVCpu->CTX_SUFF(pVM);
5271	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5272	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
5273	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
5274	&& ( !CPUMIsGuestInRealModeEx(pCtx)
5275	&& !CPUMIsGuestInV86ModeEx(pCtx)))
5276	{
5277	/* Protected mode checks */
5278	/* CS */
5279	Assert(pCtx->cs.Attr.n.u1Present);
5280	Assert(!(pCtx->cs.Attr.u & 0xf00));
5281	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
5282	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5283	\|\| !(pCtx->cs.Attr.n.u1Granularity));
5284	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
5285	\|\| (pCtx->cs.Attr.n.u1Granularity));
5286	/* CS cannot be loaded with NULL in protected mode. */
5287	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
5288	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5289	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
5290	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5291	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
5292	else
5293	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
5294	/* SS */
5295	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5296	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
5297	if ( !(pCtx->cr0 & X86_CR0_PE)
5298	\|\| pCtx->cs.Attr.n.u4Type == 3)
5299	{
5300	Assert(!pCtx->ss.Attr.n.u2Dpl);
5301	}
5302	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
5303	{
5304	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5305	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
5306	Assert(pCtx->ss.Attr.n.u1Present);
5307	Assert(!(pCtx->ss.Attr.u & 0xf00));
5308	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
5309	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
5310	\|\| !(pCtx->ss.Attr.n.u1Granularity));
5311	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
5312	\|\| (pCtx->ss.Attr.n.u1Granularity));
5313	}
5314	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
5315	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
5316	{
5317	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5318	Assert(pCtx->ds.Attr.n.u1Present);
5319	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
5320	Assert(!(pCtx->ds.Attr.u & 0xf00));
5321	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
5322	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
5323	\|\| !(pCtx->ds.Attr.n.u1Granularity));
5324	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
5325	\|\| (pCtx->ds.Attr.n.u1Granularity));
5326	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5327	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
5328	}
5329	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
5330	{
5331	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5332	Assert(pCtx->es.Attr.n.u1Present);
5333	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
5334	Assert(!(pCtx->es.Attr.u & 0xf00));
5335	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
5336	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
5337	\|\| !(pCtx->es.Attr.n.u1Granularity));
5338	Assert( !(pCtx->es.u32Limit & 0xfff00000)
5339	\|\| (pCtx->es.Attr.n.u1Granularity));
5340	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5341	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
5342	}
5343	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
5344	{
5345	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5346	Assert(pCtx->fs.Attr.n.u1Present);
5347	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
5348	Assert(!(pCtx->fs.Attr.u & 0xf00));
5349	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
5350	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
5351	\|\| !(pCtx->fs.Attr.n.u1Granularity));
5352	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
5353	\|\| (pCtx->fs.Attr.n.u1Granularity));
5354	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5355	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5356	}
5357	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
5358	{
5359	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5360	Assert(pCtx->gs.Attr.n.u1Present);
5361	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
5362	Assert(!(pCtx->gs.Attr.u & 0xf00));
5363	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
5364	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
5365	\|\| !(pCtx->gs.Attr.n.u1Granularity));
5366	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
5367	\|\| (pCtx->gs.Attr.n.u1Granularity));
5368	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5369	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5370	}
5371	/* 64-bit capable CPUs. */
5372	# if HC_ARCH_BITS == 64
5373	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5374	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
5375	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
5376	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
5377	# endif
5378	}
5379	else if ( CPUMIsGuestInV86ModeEx(pCtx)
5380	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
5381	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
5382	{
5383	/* Real and v86 mode checks. */
5384	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
5385	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
5386	if (pVmcsInfo->RealMode.fRealOnV86Active)
5387	{
5388	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
5389	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
5390	}
5391	else
5392	{
5393	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
5394	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
5395	}
5396
5397	/* CS */
5398	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
5399	Assert(pCtx->cs.u32Limit == 0xffff);
5400	Assert(u32CSAttr == 0xf3);
5401	/* SS */
5402	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
5403	Assert(pCtx->ss.u32Limit == 0xffff);
5404	Assert(u32SSAttr == 0xf3);
5405	/* DS */
5406	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
5407	Assert(pCtx->ds.u32Limit == 0xffff);
5408	Assert(u32DSAttr == 0xf3);
5409	/* ES */
5410	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
5411	Assert(pCtx->es.u32Limit == 0xffff);
5412	Assert(u32ESAttr == 0xf3);
5413	/* FS */
5414	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
5415	Assert(pCtx->fs.u32Limit == 0xffff);
5416	Assert(u32FSAttr == 0xf3);
5417	/* GS */
5418	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
5419	Assert(pCtx->gs.u32Limit == 0xffff);
5420	Assert(u32GSAttr == 0xf3);
5421	/* 64-bit capable CPUs. */
5422	# if HC_ARCH_BITS == 64
5423	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5424	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
5425	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
5426	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
5427	# endif
5428	}
5429	}
5430	#endif /* VBOX_STRICT */
5431
5432
5433	/**
5434	* Exports a guest segment register into the guest-state area in the VMCS.
5435	*
5436	* @returns VBox status code.
5437	* @param pVCpu The cross context virtual CPU structure.
5438	* @param pVmcsInfo The VMCS info. object.
5439	* @param iSegReg The segment register number (X86_SREG_XXX).
5440	* @param pSelReg Pointer to the segment selector.
5441	*
5442	* @remarks No-long-jump zone!!!
5443	*/
5444	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
5445	{
5446	Assert(iSegReg < X86_SREG_COUNT);
5447	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
5448	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
5449	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
5450	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
5451
5452	uint32_t u32Access = pSelReg->Attr.u;
5453	if (pVmcsInfo->RealMode.fRealOnV86Active)
5454	{
5455	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
5456	u32Access = 0xf3;
5457	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5458	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5459	RT_NOREF_PV(pVCpu);
5460	}
5461	else
5462	{
5463	/*
5464	* The way to differentiate between whether this is really a null selector or was just
5465	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
5466	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
5467	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
5468	* NULL selectors loaded in protected-mode have their attribute as 0.
5469	*/
5470	if (!u32Access)
5471	u32Access = X86DESCATTR_UNUSABLE;
5472	}
5473
5474	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
5475	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
5476	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
5477
5478	/*
5479	* Commit it to the VMCS.
5480	*/
5481	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
5482	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
5483	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
5484	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
5485	AssertRCReturn(rc, rc);
5486	return rc;
5487	}
5488
5489
5490	/**
5491	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
5492	* area in the VMCS.
5493	*
5494	* @returns VBox status code.
5495	* @param pVCpu The cross context virtual CPU structure.
5496	* @param pVmxTransient The VMX-transient structure.
5497	*
5498	* @remarks Will import guest CR0 on strict builds during validation of
5499	* segments.
5500	* @remarks No-long-jump zone!!!
5501	*/
5502	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5503	{
5504	int rc = VERR_INTERNAL_ERROR_5;
5505	PVM pVM = pVCpu->CTX_SUFF(pVM);
5506	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5507	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5508
5509	/*
5510	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
5511	*/
5512	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
5513	{
5514	#ifdef VBOX_WITH_REM
5515	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
5516	{
5517	Assert(!pVmxTransient->fIsNestedGuest);
5518	Assert(pVM->hm.s.vmx.pRealModeTSS);
5519	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
5520	if ( pVmcsInfo->fWasInRealMode
5521	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
5522	{
5523	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
5524	in real-mode (e.g. OpenBSD 4.0) */
5525	REMFlushTBs(pVM);
5526	Log4Func(("Switch to protected mode detected!\n"));
5527	pVmcsInfo->fWasInRealMode = false;
5528	}
5529	}
5530	#endif
5531	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
5532	{
5533	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
5534	if (pVmcsInfo->RealMode.fRealOnV86Active)
5535	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
5536	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
5537	AssertRCReturn(rc, rc);
5538	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
5539	}
5540
5541	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
5542	{
5543	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
5544	if (pVmcsInfo->RealMode.fRealOnV86Active)
5545	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
5546	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
5547	AssertRCReturn(rc, rc);
5548	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
5549	}
5550
5551	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
5552	{
5553	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
5554	if (pVmcsInfo->RealMode.fRealOnV86Active)
5555	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
5556	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
5557	AssertRCReturn(rc, rc);
5558	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
5559	}
5560
5561	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
5562	{
5563	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
5564	if (pVmcsInfo->RealMode.fRealOnV86Active)
5565	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
5566	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
5567	AssertRCReturn(rc, rc);
5568	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
5569	}
5570
5571	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
5572	{
5573	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
5574	if (pVmcsInfo->RealMode.fRealOnV86Active)
5575	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
5576	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
5577	AssertRCReturn(rc, rc);
5578	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
5579	}
5580
5581	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
5582	{
5583	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
5584	if (pVmcsInfo->RealMode.fRealOnV86Active)
5585	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
5586	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
5587	AssertRCReturn(rc, rc);
5588	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
5589	}
5590
5591	#ifdef VBOX_STRICT
5592	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
5593	#endif
5594	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
5595	pCtx->cs.Attr.u));
5596	}
5597
5598	/*
5599	* Guest TR.
5600	*/
5601	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
5602	{
5603	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
5604
5605	/*
5606	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
5607	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
5608	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
5609	*/
5610	uint16_t u16Sel;
5611	uint32_t u32Limit;
5612	uint64_t u64Base;
5613	uint32_t u32AccessRights;
5614	if (!pVmcsInfo->RealMode.fRealOnV86Active)
5615	{
5616	u16Sel = pCtx->tr.Sel;
5617	u32Limit = pCtx->tr.u32Limit;
5618	u64Base = pCtx->tr.u64Base;
5619	u32AccessRights = pCtx->tr.Attr.u;
5620	}
5621	else
5622	{
5623	Assert(!pVmxTransient->fIsNestedGuest);
5624	Assert(pVM->hm.s.vmx.pRealModeTSS);
5625	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
5626
5627	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
5628	RTGCPHYS GCPhys;
5629	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
5630	AssertRCReturn(rc, rc);
5631
5632	X86DESCATTR DescAttr;
5633	DescAttr.u = 0;
5634	DescAttr.n.u1Present = 1;
5635	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
5636
5637	u16Sel = 0;
5638	u32Limit = HM_VTX_TSS_SIZE;
5639	u64Base = GCPhys;
5640	u32AccessRights = DescAttr.u;
5641	}
5642
5643	/* Validate. */
5644	Assert(!(u16Sel & RT_BIT(2)));
5645	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
5646	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
5647	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
5648	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
5649	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
5650	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
5651	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
5652	Assert( (u32Limit & 0xfff) == 0xfff
5653	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
5654	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
5655	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
5656
5657	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
5658	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
5659	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
5660	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
5661	AssertRCReturn(rc, rc);
5662
5663	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
5664	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
5665	}
5666
5667	/*
5668	* Guest GDTR.
5669	*/
5670	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
5671	{
5672	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
5673
5674	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
5675	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
5676	AssertRCReturn(rc, rc);
5677
5678	/* Validate. */
5679	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5680
5681	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
5682	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
5683	}
5684
5685	/*
5686	* Guest LDTR.
5687	*/
5688	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
5689	{
5690	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
5691
5692	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
5693	uint32_t u32Access;
5694	if ( !pVmxTransient->fIsNestedGuest
5695	&& !pCtx->ldtr.Attr.u)
5696	u32Access = X86DESCATTR_UNUSABLE;
5697	else
5698	u32Access = pCtx->ldtr.Attr.u;
5699
5700	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
5701	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
5702	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
5703	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
5704	AssertRCReturn(rc, rc);
5705
5706	/* Validate. */
5707	if (!(u32Access & X86DESCATTR_UNUSABLE))
5708	{
5709	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
5710	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
5711	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
5712	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
5713	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
5714	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
5715	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
5716	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
5717	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
5718	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
5719	}
5720
5721	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
5722	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
5723	}
5724
5725	/*
5726	* Guest IDTR.
5727	*/
5728	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
5729	{
5730	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
5731
5732	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
5733	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
5734	AssertRCReturn(rc, rc);
5735
5736	/* Validate. */
5737	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5738
5739	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
5740	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
5741	}
5742
5743	return VINF_SUCCESS;
5744	}
5745
5746
5747	/**
5748	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
5749	* areas.
5750	*
5751	* These MSRs will automatically be loaded to the host CPU on every successful
5752	* VM-entry and stored from the host CPU on every successful VM-exit.
5753	*
5754	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
5755	* actual host MSR values are not- updated here for performance reasons. See
5756	* hmR0VmxExportHostMsrs().
5757	*
5758	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
5759	*
5760	* @returns VBox status code.
5761	* @param pVCpu The cross context virtual CPU structure.
5762	* @param pVmxTransient The VMX-transient structure.
5763	*
5764	* @remarks No-long-jump zone!!!
5765	*/
5766	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5767	{
5768	AssertPtr(pVCpu);
5769	AssertPtr(pVmxTransient);
5770
5771	PVM pVM = pVCpu->CTX_SUFF(pVM);
5772	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5773
5774	/*
5775	* MSRs that we use the auto-load/store MSR area in the VMCS.
5776	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
5777	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
5778	*
5779	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
5780	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
5781	* emulation, nothing to do here.
5782	*/
5783	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
5784	{
5785	if ( !pVmxTransient->fIsNestedGuest
5786	&& pVM->hm.s.fAllow64BitGuests)
5787	{
5788	#if HC_ARCH_BITS == 32
5789	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
5790	Assert(!pVmxTransient->fIsNestedGuest);
5791
5792	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
5793	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
5794	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
5795	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
5796	AssertRCReturn(rc, rc);
5797	#endif
5798	}
5799	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
5800	}
5801
5802	/*
5803	* Guest Sysenter MSRs.
5804	*/
5805	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
5806	{
5807	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
5808
5809	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
5810	{
5811	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
5812	AssertRCReturn(rc, rc);
5813	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
5814	}
5815
5816	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
5817	{
5818	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
5819	AssertRCReturn(rc, rc);
5820	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
5821	}
5822
5823	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
5824	{
5825	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
5826	AssertRCReturn(rc, rc);
5827	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
5828	}
5829	}
5830
5831	/*
5832	* Guest/host EFER MSR.
5833	*/
5834	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
5835	{
5836	/* Whether we are using the VMCS to swap the EFER MSR must have been
5837	determined earlier while exporting VM-entry/VM-exit controls. */
5838	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
5839	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
5840
5841	if (hmR0VmxShouldSwapEferMsr(pVCpu))
5842	{
5843	/*
5844	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
5845	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
5846	*/
5847	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
5848	{
5849	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
5850	AssertRCReturn(rc, rc);
5851	}
5852	else
5853	{
5854	/*
5855	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
5856	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
5857	*/
5858	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
5859	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5860	AssertRCReturn(rc, rc);
5861	}
5862	}
5863	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
5864	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
5865
5866	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
5867	}
5868
5869	/*
5870	* Other MSRs.
5871	* Speculation Control (R/W).
5872	*/
5873	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
5874	{
5875	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
5876	if (pVM->cpum.ro.GuestFeatures.fIbrs)
5877	{
5878	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
5879	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5880	AssertRCReturn(rc, rc);
5881	}
5882	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
5883	}
5884
5885	return VINF_SUCCESS;
5886	}
5887
5888
5889	/**
5890	* Selects up the appropriate function to run guest code.
5891	*
5892	* @returns VBox status code.
5893	* @param pVCpu The cross context virtual CPU structure.
5894	* @param pVmxTransient The VMX-transient structure.
5895	*
5896	* @remarks No-long-jump zone!!!
5897	*/
5898	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5899	{
5900	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5901	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5902
5903	if (CPUMIsGuestInLongModeEx(pCtx))
5904	{
5905	#ifndef VBOX_ENABLE_64_BITS_GUESTS
5906	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5907	#endif
5908	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
5909	#if HC_ARCH_BITS == 32
5910	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
5911	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
5912	{
5913	#ifdef VBOX_STRICT
5914	if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5915	{
5916	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5917	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5918	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5919	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5920	("fCtxChanged=%#RX64\n", fCtxChanged));
5921	}
5922	#endif
5923	pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
5924
5925	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
5926	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
5927	pVmcsInfo->fSwitchedTo64on32 = true;
5928	Log4Func(("Selected 64-bit switcher\n"));
5929	}
5930	#else
5931	/* 64-bit host. */
5932	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
5933	#endif
5934	}
5935	else
5936	{
5937	/* Guest is not in long mode, use the 32-bit handler. */
5938	#if HC_ARCH_BITS == 32
5939	if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
5940	&& !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
5941	&& pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5942	{
5943	# ifdef VBOX_STRICT
5944	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5945	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5946	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5947	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5948	("fCtxChanged=%#RX64\n", fCtxChanged));
5949	# endif
5950	}
5951	# ifdef VBOX_ENABLE_64_BITS_GUESTS
5952	/*
5953	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
5954	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
5955	* switcher flag now because we know the guest is in a sane state where it's safe
5956	* to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
5957	* the much faster 32-bit switcher again.
5958	*/
5959	if (!pVmcsInfo->fSwitchedTo64on32)
5960	{
5961	if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
5962	Log4Func(("Selected 32-bit switcher\n"));
5963	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5964	}
5965	else
5966	{
5967	Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
5968	if ( pVmcsInfo->RealMode.fRealOnV86Active
5969	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
5970	{
5971	pVmcsInfo->fSwitchedTo64on32 = false;
5972	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5973	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
5974	\| HM_CHANGED_VMX_ENTRY_EXIT_CTLS
5975	\| HM_CHANGED_HOST_CONTEXT);
5976	Log4Func(("Selected 32-bit switcher (safe)\n"));
5977	}
5978	}
5979	# else
5980	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5981	# endif
5982	#else
5983	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5984	#endif
5985	}
5986	Assert(pVmcsInfo->pfnStartVM);
5987	return VINF_SUCCESS;
5988	}
5989
5990
5991	/**
5992	* Wrapper for running the guest code in VT-x.
5993	*
5994	* @returns VBox status code, no informational status codes.
5995	* @param pVCpu The cross context virtual CPU structure.
5996	* @param pVmxTransient The VMX-transient structure.
5997	*
5998	* @remarks No-long-jump zone!!!
5999	*/
6000	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6001	{
6002	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6003	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6004	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6005
6006	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6007
6008	/*
6009	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6010	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6011	* callee-saved and thus the need for this XMM wrapper.
6012	*
6013	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6014	*/
6015	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6016	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6017	PVM pVM = pVCpu->CTX_SUFF(pVM);
6018	#ifdef VBOX_WITH_KERNEL_USING_XMM
6019	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6020	#else
6021	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
6022	#endif
6023	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6024	return rc;
6025	}
6026
6027
6028	/**
6029	* Reports world-switch error and dumps some useful debug info.
6030	*
6031	* @param pVCpu The cross context virtual CPU structure.
6032	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6033	* @param pVmxTransient The VMX-transient structure (only
6034	* exitReason updated).
6035	*/
6036	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6037	{
6038	Assert(pVCpu);
6039	Assert(pVmxTransient);
6040	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6041
6042	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6043	switch (rcVMRun)
6044	{
6045	case VERR_VMX_INVALID_VMXON_PTR:
6046	AssertFailed();
6047	break;
6048	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6049	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6050	{
6051	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6052	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6053	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
6054	AssertRC(rc);
6055
6056	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6057	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6058	Cannot do it here as we may have been long preempted. */
6059
6060	#ifdef VBOX_STRICT
6061	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6062	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6063	pVmxTransient->uExitReason));
6064	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6065	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6066	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6067	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6068	else
6069	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6070	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6071	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6072
6073	/* VMX control bits. */
6074	uint32_t u32Val;
6075	uint64_t u64Val;
6076	RTHCUINTREG uHCReg;
6077	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
6078	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
6079	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
6080	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
6081	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
6082	{
6083	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
6084	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
6085	}
6086	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
6087	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
6088	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
6089	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
6090	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
6091	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
6092	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
6093	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
6094	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
6095	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
6096	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
6097	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
6098	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
6099	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
6100	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
6101	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
6102	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6103	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
6104	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6105	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
6106	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
6107	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
6108	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
6109	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
6110	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
6111	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
6112	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
6113	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
6114	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
6115	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6116	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
6117	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
6118	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
6119	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6120	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6121	{
6122	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
6123	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
6124	}
6125
6126	/* Guest bits. */
6127	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
6128	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
6129	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
6130	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
6131	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
6132	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
6133	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
6134	{
6135	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
6136	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
6137	}
6138
6139	/* Host bits. */
6140	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
6141	Log4(("Host CR0 %#RHr\n", uHCReg));
6142	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
6143	Log4(("Host CR3 %#RHr\n", uHCReg));
6144	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
6145	Log4(("Host CR4 %#RHr\n", uHCReg));
6146
6147	RTGDTR HostGdtr;
6148	PCX86DESCHC pDesc;
6149	ASMGetGDTR(&HostGdtr);
6150	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
6151	Log4(("Host CS %#08x\n", u32Val));
6152	if (u32Val < HostGdtr.cbGdt)
6153	{
6154	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6155	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
6156	}
6157
6158	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
6159	Log4(("Host DS %#08x\n", u32Val));
6160	if (u32Val < HostGdtr.cbGdt)
6161	{
6162	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6163	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
6164	}
6165
6166	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
6167	Log4(("Host ES %#08x\n", u32Val));
6168	if (u32Val < HostGdtr.cbGdt)
6169	{
6170	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6171	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
6172	}
6173
6174	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
6175	Log4(("Host FS %#08x\n", u32Val));
6176	if (u32Val < HostGdtr.cbGdt)
6177	{
6178	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6179	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
6180	}
6181
6182	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
6183	Log4(("Host GS %#08x\n", u32Val));
6184	if (u32Val < HostGdtr.cbGdt)
6185	{
6186	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6187	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
6188	}
6189
6190	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
6191	Log4(("Host SS %#08x\n", u32Val));
6192	if (u32Val < HostGdtr.cbGdt)
6193	{
6194	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6195	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
6196	}
6197
6198	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
6199	Log4(("Host TR %#08x\n", u32Val));
6200	if (u32Val < HostGdtr.cbGdt)
6201	{
6202	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6203	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
6204	}
6205
6206	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
6207	Log4(("Host TR Base %#RHv\n", uHCReg));
6208	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
6209	Log4(("Host GDTR Base %#RHv\n", uHCReg));
6210	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
6211	Log4(("Host IDTR Base %#RHv\n", uHCReg));
6212	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
6213	Log4(("Host SYSENTER CS %#08x\n", u32Val));
6214	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
6215	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
6216	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
6217	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
6218	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
6219	Log4(("Host RSP %#RHv\n", uHCReg));
6220	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
6221	Log4(("Host RIP %#RHv\n", uHCReg));
6222	# if HC_ARCH_BITS == 64
6223	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6224	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6225	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6226	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6227	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6228	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6229	# endif
6230	#endif /* VBOX_STRICT */
6231	break;
6232	}
6233
6234	default:
6235	/* Impossible */
6236	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6237	break;
6238	}
6239	}
6240
6241
6242	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
6243	# ifndef VMX_USE_CACHED_VMCS_ACCESSES
6244	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
6245	# endif
6246
6247	/**
6248	* Initialize the VMCS-Read cache.
6249	*
6250	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
6251	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
6252	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
6253	* (those that have a 32-bit FULL & HIGH part).
6254	*
6255	* @param pVCpu The cross context virtual CPU structure.
6256	*/
6257	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
6258	{
6259	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
6260	do { \
6261	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
6262	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
6263	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
6264	++cReadFields; \
6265	} while (0)
6266
6267	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6268	uint32_t cReadFields = 0;
6269
6270	/*
6271	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
6272	* and serve to indicate exceptions to the rules.
6273	*/
6274
6275	/* Guest-natural selector base fields. */
6276	#if 0
6277	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
6278	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
6279	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
6280	#endif
6281	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
6282	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
6283	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
6284	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
6285	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
6286	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
6287	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
6288	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
6289	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
6290	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
6291	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
6292	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
6293	#if 0
6294	/* Unused natural width guest-state fields. */
6295	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
6296	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
6297	#endif
6298	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
6299	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
6300
6301	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
6302	these 64-bit fields (using "FULL" and "HIGH" fields). */
6303	#if 0
6304	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
6305	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
6306	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
6307	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
6308	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
6309	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
6310	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
6311	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
6312	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
6313	#endif
6314
6315	/* Natural width guest-state fields. */
6316	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
6317	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
6318
6319	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6320	{
6321	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
6322	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
6323	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
6324	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
6325	}
6326	else
6327	{
6328	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
6329	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
6330	}
6331
6332	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
6333	}
6334
6335
6336	/**
6337	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
6338	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
6339	* darwin, running 64-bit guests).
6340	*
6341	* @returns VBox status code.
6342	* @param pVCpu The cross context virtual CPU structure.
6343	* @param idxField The VMCS field encoding.
6344	* @param u64Val 16, 32 or 64-bit value.
6345	*/
6346	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6347	{
6348	int rc;
6349	switch (idxField)
6350	{
6351	/*
6352	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
6353	*/
6354	/* 64-bit Control fields. */
6355	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
6356	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
6357	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
6358	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
6359	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
6360	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
6361	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
6362	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
6363	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
6364	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
6365	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
6366	case VMX_VMCS64_CTRL_EPTP_FULL:
6367	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
6368	/* 64-bit Guest-state fields. */
6369	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
6370	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
6371	case VMX_VMCS64_GUEST_PAT_FULL:
6372	case VMX_VMCS64_GUEST_EFER_FULL:
6373	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
6374	case VMX_VMCS64_GUEST_PDPTE0_FULL:
6375	case VMX_VMCS64_GUEST_PDPTE1_FULL:
6376	case VMX_VMCS64_GUEST_PDPTE2_FULL:
6377	case VMX_VMCS64_GUEST_PDPTE3_FULL:
6378	/* 64-bit Host-state fields. */
6379	case VMX_VMCS64_HOST_PAT_FULL:
6380	case VMX_VMCS64_HOST_EFER_FULL:
6381	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
6382	{
6383	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6384	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
6385	break;
6386	}
6387
6388	/*
6389	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
6390	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
6391	*/
6392	/* Natural-width Guest-state fields. */
6393	case VMX_VMCS_GUEST_CR3:
6394	case VMX_VMCS_GUEST_ES_BASE:
6395	case VMX_VMCS_GUEST_CS_BASE:
6396	case VMX_VMCS_GUEST_SS_BASE:
6397	case VMX_VMCS_GUEST_DS_BASE:
6398	case VMX_VMCS_GUEST_FS_BASE:
6399	case VMX_VMCS_GUEST_GS_BASE:
6400	case VMX_VMCS_GUEST_LDTR_BASE:
6401	case VMX_VMCS_GUEST_TR_BASE:
6402	case VMX_VMCS_GUEST_GDTR_BASE:
6403	case VMX_VMCS_GUEST_IDTR_BASE:
6404	case VMX_VMCS_GUEST_RSP:
6405	case VMX_VMCS_GUEST_RIP:
6406	case VMX_VMCS_GUEST_SYSENTER_ESP:
6407	case VMX_VMCS_GUEST_SYSENTER_EIP:
6408	{
6409	if (!(RT_HI_U32(u64Val)))
6410	{
6411	/* If this field is 64-bit, VT-x will zero out the top bits. */
6412	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6413	}
6414	else
6415	{
6416	/* Assert that only the 32->64 switcher case should ever come here. */
6417	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
6418	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
6419	}
6420	break;
6421	}
6422
6423	default:
6424	{
6425	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
6426	pVCpu->hm.s.u32HMError = idxField;
6427	rc = VERR_INVALID_PARAMETER;
6428	break;
6429	}
6430	}
6431	AssertRCReturn(rc, rc);
6432	return rc;
6433	}
6434
6435
6436	/**
6437	* Queue up a VMWRITE by using the VMCS write cache.
6438	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
6439	*
6440	* @param pVCpu The cross context virtual CPU structure.
6441	* @param idxField The VMCS field encoding.
6442	* @param u64Val 16, 32 or 64-bit value.
6443	*/
6444	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6445	{
6446	AssertPtr(pVCpu);
6447	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6448
6449	AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
6450	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
6451
6452	/* Make sure there are no duplicates. */
6453	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
6454	{
6455	if (pCache->Write.aField[i] == idxField)
6456	{
6457	pCache->Write.aFieldVal[i] = u64Val;
6458	return VINF_SUCCESS;
6459	}
6460	}
6461
6462	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
6463	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
6464	pCache->Write.cValidEntries++;
6465	return VINF_SUCCESS;
6466	}
6467	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
6468
6469
6470	/**
6471	* Sets up the usage of TSC-offsetting and updates the VMCS.
6472	*
6473	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6474	* VMX-preemption timer.
6475	*
6476	* @returns VBox status code.
6477	* @param pVCpu The cross context virtual CPU structure.
6478	* @param pVmxTransient The VMX-transient structure.
6479	*
6480	* @remarks No-long-jump zone!!!
6481	*/
6482	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6483	{
6484	bool fOffsettedTsc;
6485	bool fParavirtTsc;
6486	uint64_t uTscOffset;
6487	PVM pVM = pVCpu->CTX_SUFF(pVM);
6488	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6489
6490	if (pVM->hm.s.vmx.fUsePreemptTimer)
6491	{
6492	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6493
6494	/* Make sure the returned values have sane upper and lower boundaries. */
6495	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6496	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6497	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6498	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6499
6500	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6501	* preemption timers here. We probably need to clamp the preemption timer,
6502	* after converting the timer value to the host. */
6503	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6504	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6505	AssertRC(rc);
6506	}
6507	else
6508	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6509
6510	if (fParavirtTsc)
6511	{
6512	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6513	information before every VM-entry, hence disable it for performance sake. */
6514	#if 0
6515	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
6516	AssertRC(rc);
6517	#endif
6518	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
6519	}
6520
6521	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
6522	if ( fOffsettedTsc
6523	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
6524	{
6525	if (pVmxTransient->fIsNestedGuest)
6526	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
6527	if (pVmcsInfo->u64TscOffset != uTscOffset)
6528	{
6529	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
6530	AssertRC(rc);
6531	pVmcsInfo->u64TscOffset = uTscOffset;
6532	}
6533
6534	if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
6535	{
6536	uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
6537	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6538	AssertRC(rc);
6539	pVmcsInfo->u32ProcCtls = uProcCtls;
6540	}
6541	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
6542	}
6543	else
6544	{
6545	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
6546	if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
6547	{
6548	uProcCtls \|= VMX_PROC_CTLS_RDTSC_EXIT;
6549	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6550	AssertRC(rc);
6551	pVmcsInfo->u32ProcCtls = uProcCtls;
6552	}
6553	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
6554	}
6555	}
6556
6557
6558	/**
6559	* Gets the IEM exception flags for the specified vector and IDT vectoring /
6560	* VM-exit interruption info type.
6561	*
6562	* @returns The IEM exception flags.
6563	* @param uVector The event vector.
6564	* @param uVmxEventType The VMX event type.
6565	*
6566	* @remarks This function currently only constructs flags required for
6567	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
6568	* and CR2 aspects of an exception are not included).
6569	*/
6570	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
6571	{
6572	uint32_t fIemXcptFlags;
6573	switch (uVmxEventType)
6574	{
6575	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6576	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6577	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6578	break;
6579
6580	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6581	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6582	break;
6583
6584	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6585	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
6586	break;
6587
6588	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
6589	{
6590	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6591	if (uVector == X86_XCPT_BP)
6592	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
6593	else if (uVector == X86_XCPT_OF)
6594	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
6595	else
6596	{
6597	fIemXcptFlags = 0;
6598	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
6599	}
6600	break;
6601	}
6602
6603	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6604	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6605	break;
6606
6607	default:
6608	fIemXcptFlags = 0;
6609	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
6610	break;
6611	}
6612	return fIemXcptFlags;
6613	}
6614
6615
6616	/**
6617	* Sets an event as a pending event to be injected into the guest.
6618	*
6619	* @param pVCpu The cross context virtual CPU structure.
6620	* @param u32IntInfo The VM-entry interruption-information field.
6621	* @param cbInstr The VM-entry instruction length in bytes (for software
6622	* interrupts, exceptions and privileged software
6623	* exceptions).
6624	* @param u32ErrCode The VM-entry exception error code.
6625	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
6626	* page-fault.
6627	*/
6628	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
6629	RTGCUINTPTR GCPtrFaultAddress)
6630	{
6631	Assert(!pVCpu->hm.s.Event.fPending);
6632	pVCpu->hm.s.Event.fPending = true;
6633	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
6634	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
6635	pVCpu->hm.s.Event.cbInstr = cbInstr;
6636	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
6637	}
6638
6639
6640	/**
6641	* Sets an external interrupt as pending-for-injection into the VM.
6642	*
6643	* @param pVCpu The cross context virtual CPU structure.
6644	* @param u8Interrupt The external interrupt vector.
6645	*/
6646	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
6647	{
6648	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
6649	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6650	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6651	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6652	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6653	}
6654
6655
6656	/**
6657	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
6658	*
6659	* @param pVCpu The cross context virtual CPU structure.
6660	*/
6661	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
6662	{
6663	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
6664	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
6665	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6666	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6667	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6668	}
6669
6670
6671	/**
6672	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
6673	*
6674	* @param pVCpu The cross context virtual CPU structure.
6675	*/
6676	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
6677	{
6678	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
6679	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6680	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6681	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6682	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6683	}
6684
6685
6686	/**
6687	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
6688	*
6689	* @param pVCpu The cross context virtual CPU structure.
6690	*/
6691	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
6692	{
6693	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
6694	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6695	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6696	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6697	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6698	}
6699
6700
6701	/**
6702	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
6703	*
6704	* @param pVCpu The cross context virtual CPU structure.
6705	*/
6706	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
6707	{
6708	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
6709	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6710	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6711	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6712	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6713	}
6714
6715
6716	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6717	/**
6718	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
6719	*
6720	* @param pVCpu The cross context virtual CPU structure.
6721	* @param u32ErrCode The error code for the general-protection exception.
6722	*/
6723	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
6724	{
6725	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
6726	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6727	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6728	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6729	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6730	}
6731
6732
6733	/**
6734	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
6735	*
6736	* @param pVCpu The cross context virtual CPU structure.
6737	* @param u32ErrCode The error code for the stack exception.
6738	*/
6739	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
6740	{
6741	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
6742	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6743	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6744	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6745	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6746	}
6747
6748
6749	/**
6750	* Decodes the memory operand of an instruction that caused a VM-exit.
6751	*
6752	* The VM-exit qualification field provides the displacement field for memory
6753	* operand instructions, if any.
6754	*
6755	* @returns Strict VBox status code (i.e. informational status codes too).
6756	* @retval VINF_SUCCESS if the operand was successfully decoded.
6757	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
6758	* operand.
6759	* @param pVCpu The cross context virtual CPU structure.
6760	* @param uExitInstrInfo The VM-exit instruction information field.
6761	* @param enmMemAccess The memory operand's access type (read or write).
6762	* @param GCPtrDisp The instruction displacement field, if any. For
6763	* RIP-relative addressing pass RIP + displacement here.
6764	* @param pGCPtrMem Where to store the effective destination memory address.
6765	*/
6766	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
6767	PRTGCPTR pGCPtrMem)
6768	{
6769	Assert(pGCPtrMem);
6770	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
6771	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
6772	\| CPUMCTX_EXTRN_CR0);
6773
6774	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
6775	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
6776	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
6777
6778	VMXEXITINSTRINFO ExitInstrInfo;
6779	ExitInstrInfo.u = uExitInstrInfo;
6780	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
6781	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
6782	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
6783	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
6784	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
6785	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
6786	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
6787	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
6788	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
6789
6790	/*
6791	* Validate instruction information.
6792	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
6793	*/
6794	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
6795	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
6796	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
6797	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
6798	AssertLogRelMsgReturn(fIsMemOperand,
6799	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
6800
6801	/*
6802	* Compute the complete effective address.
6803	*
6804	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
6805	* See AMD spec. 4.5.2 "Segment Registers".
6806	*/
6807	RTGCPTR GCPtrMem = GCPtrDisp;
6808	if (fBaseRegValid)
6809	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
6810	if (fIdxRegValid)
6811	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
6812
6813	RTGCPTR const GCPtrOff = GCPtrMem;
6814	if ( !fIsLongMode
6815	\|\| iSegReg >= X86_SREG_FS)
6816	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
6817	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
6818
6819	/*
6820	* Validate effective address.
6821	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
6822	*/
6823	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
6824	Assert(cbAccess > 0);
6825	if (fIsLongMode)
6826	{
6827	if (X86_IS_CANONICAL(GCPtrMem))
6828	{
6829	*pGCPtrMem = GCPtrMem;
6830	return VINF_SUCCESS;
6831	}
6832
6833	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
6834	* "Data Limit Checks in 64-bit Mode". */
6835	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
6836	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6837	return VINF_HM_PENDING_XCPT;
6838	}
6839
6840	/*
6841	* This is a watered down version of iemMemApplySegment().
6842	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
6843	* and segment CPL/DPL checks are skipped.
6844	*/
6845	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
6846	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
6847	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
6848
6849	/* Check if the segment is present and usable. */
6850	if ( pSel->Attr.n.u1Present
6851	&& !pSel->Attr.n.u1Unusable)
6852	{
6853	Assert(pSel->Attr.n.u1DescType);
6854	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
6855	{
6856	/* Check permissions for the data segment. */
6857	if ( enmMemAccess == VMXMEMACCESS_WRITE
6858	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
6859	{
6860	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6861	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
6862	return VINF_HM_PENDING_XCPT;
6863	}
6864
6865	/* Check limits if it's a normal data segment. */
6866	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
6867	{
6868	if ( GCPtrFirst32 > pSel->u32Limit
6869	\|\| GCPtrLast32 > pSel->u32Limit)
6870	{
6871	Log4Func(("Data segment limit exceeded."
6872	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6873	GCPtrLast32, pSel->u32Limit));
6874	if (iSegReg == X86_SREG_SS)
6875	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6876	else
6877	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6878	return VINF_HM_PENDING_XCPT;
6879	}
6880	}
6881	else
6882	{
6883	/* Check limits if it's an expand-down data segment.
6884	Note! The upper boundary is defined by the B bit, not the G bit! */
6885	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
6886	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
6887	{
6888	Log4Func(("Expand-down data segment limit exceeded."
6889	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6890	GCPtrLast32, pSel->u32Limit));
6891	if (iSegReg == X86_SREG_SS)
6892	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6893	else
6894	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6895	return VINF_HM_PENDING_XCPT;
6896	}
6897	}
6898	}
6899	else
6900	{
6901	/* Check permissions for the code segment. */
6902	if ( enmMemAccess == VMXMEMACCESS_WRITE
6903	\|\| ( enmMemAccess == VMXMEMACCESS_READ
6904	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
6905	{
6906	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6907	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6908	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6909	return VINF_HM_PENDING_XCPT;
6910	}
6911
6912	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6913	if ( GCPtrFirst32 > pSel->u32Limit
6914	\|\| GCPtrLast32 > pSel->u32Limit)
6915	{
6916	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6917	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6918	if (iSegReg == X86_SREG_SS)
6919	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6920	else
6921	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6922	return VINF_HM_PENDING_XCPT;
6923	}
6924	}
6925	}
6926	else
6927	{
6928	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6929	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6930	return VINF_HM_PENDING_XCPT;
6931	}
6932
6933	*pGCPtrMem = GCPtrMem;
6934	return VINF_SUCCESS;
6935	}
6936
6937
6938	/**
6939	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6940	* guest attempting to execute a VMX instruction.
6941	*
6942	* @returns Strict VBox status code (i.e. informational status codes too).
6943	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
6944	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
6945	*
6946	* @param pVCpu The cross context virtual CPU structure.
6947	* @param uExitReason The VM-exit reason.
6948	*
6949	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
6950	* @remarks No-long-jump zone!!!
6951	*/
6952	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
6953	{
6954	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
6955	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
6956
6957	if ( CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx)
6958	\|\| ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6959	&& !CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)))
6960	{
6961	Log4Func(("In real/v86-mode or long-mode outside 64-bit code segment -> #UD\n"));
6962	hmR0VmxSetPendingXcptUD(pVCpu);
6963	return VINF_HM_PENDING_XCPT;
6964	}
6965
6966	if (uExitReason == VMX_EXIT_VMXON)
6967	{
6968	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
6969
6970	/*
6971	* We check CR4.VMXE because it is required to be always set while in VMX operation
6972	* by physical CPUs and our CR4 read shadow is only consulted when executing specific
6973	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
6974	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
6975	*/
6976	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
6977	{
6978	Log4Func(("CR4.VMXE is not set -> #UD\n"));
6979	hmR0VmxSetPendingXcptUD(pVCpu);
6980	return VINF_HM_PENDING_XCPT;
6981	}
6982	}
6983	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
6984	{
6985	/*
6986	* The guest has not entered VMX operation but attempted to execute a VMX instruction
6987	* (other than VMXON), we need to raise a #UD.
6988	*/
6989	Log4Func(("Not in VMX root mode -> #UD\n"));
6990	hmR0VmxSetPendingXcptUD(pVCpu);
6991	return VINF_HM_PENDING_XCPT;
6992	}
6993
6994	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
6995	{
6996	/*
6997	* The nested-guest attempted to execute a VMX instruction, cause a VM-exit and let
6998	* the guest hypervisor deal with it.
6999	*/
7000	/** @todo NSTVMX: Trigger a VM-exit */
7001	}
7002
7003	/*
7004	* VMX instructions require CPL 0 except in VMX non-root mode where the VM-exit intercept
7005	* (above) takes preceedence over the CPL check.
7006	*/
7007	if (CPUMGetGuestCPL(pVCpu) > 0)
7008	{
7009	Log4Func(("CPL > 0 -> #GP(0)\n"));
7010	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7011	return VINF_HM_PENDING_XCPT;
7012	}
7013
7014	return VINF_SUCCESS;
7015	}
7016	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7017
7018
7019	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7020	{
7021	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7022
7023	/*
7024	* If VT-x marks the segment as unusable, most other bits remain undefined:
7025	* - For CS the L, D and G bits have meaning.
7026	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7027	* - For the remaining data segments no bits are defined.
7028	*
7029	* The present bit and the unusable bit has been observed to be set at the
7030	* same time (the selector was supposed to be invalid as we started executing
7031	* a V8086 interrupt in ring-0).
7032	*
7033	* What should be important for the rest of the VBox code, is that the P bit is
7034	* cleared. Some of the other VBox code recognizes the unusable bit, but
7035	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7036	* safe side here, we'll strip off P and other bits we don't care about. If
7037	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7038	*
7039	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7040	*/
7041	#ifdef VBOX_STRICT
7042	uint32_t const uAttr = pSelReg->Attr.u;
7043	#endif
7044
7045	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7046	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7047	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7048
7049	#ifdef VBOX_STRICT
7050	VMMRZCallRing3Disable(pVCpu);
7051	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7052	# ifdef DEBUG_bird
7053	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7054	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7055	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7056	# endif
7057	VMMRZCallRing3Enable(pVCpu);
7058	NOREF(uAttr);
7059	#endif
7060	RT_NOREF2(pVCpu, idxSel);
7061	}
7062
7063
7064	/**
7065	* Imports a guest segment register from the current VMCS into the guest-CPU
7066	* context.
7067	*
7068	* @returns VBox status code.
7069	* @param pVCpu The cross context virtual CPU structure.
7070	* @param iSegReg The segment register number (X86_SREG_XXX).
7071	*
7072	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7073	* do not log!
7074	*/
7075	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7076	{
7077	Assert(iSegReg < X86_SREG_COUNT);
7078
7079	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7080	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7081	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7082	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
7083	uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
7084	#else
7085	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7086	#endif
7087	uint64_t u64Base;
7088	uint32_t u32Sel, u32Limit, u32Attr;
7089	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7090	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7091	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7092	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
7093	if (RT_SUCCESS(rc))
7094	{
7095	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7096	pSelReg->Sel = u32Sel;
7097	pSelReg->ValidSel = u32Sel;
7098	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7099	pSelReg->u32Limit = u32Limit;
7100	pSelReg->u64Base = u64Base;
7101	pSelReg->Attr.u = u32Attr;
7102	if (u32Attr & X86DESCATTR_UNUSABLE)
7103	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7104	}
7105	return rc;
7106	}
7107
7108
7109	/**
7110	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7111	*
7112	* @returns VBox status code.
7113	* @param pVCpu The cross context virtual CPU structure.
7114	*
7115	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7116	* do not log!
7117	*/
7118	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7119	{
7120	uint64_t u64Base;
7121	uint32_t u32Sel, u32Limit, u32Attr;
7122	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7123	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7124	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7125	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7126	if (RT_SUCCESS(rc))
7127	{
7128	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7129	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7130	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7131	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7132	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7133	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7134	if (u32Attr & X86DESCATTR_UNUSABLE)
7135	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7136	}
7137	return rc;
7138	}
7139
7140
7141	/**
7142	* Imports the guest TR from the current VMCS into the guest-CPU context.
7143	*
7144	* @returns VBox status code.
7145	* @param pVCpu The cross context virtual CPU structure.
7146	*
7147	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7148	* do not log!
7149	*/
7150	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7151	{
7152	uint32_t u32Sel, u32Limit, u32Attr;
7153	uint64_t u64Base;
7154	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7155	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7156	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7157	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7158	AssertRCReturn(rc, rc);
7159
7160	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7161	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7162	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7163	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7164	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7165	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7166	/* TR is the only selector that can never be unusable. */
7167	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7168	return VINF_SUCCESS;
7169	}
7170
7171
7172	/**
7173	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7174	*
7175	* @returns VBox status code.
7176	* @param pVCpu The cross context virtual CPU structure.
7177	*
7178	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7179	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7180	* instead!!!
7181	*/
7182	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7183	{
7184	uint64_t u64Val;
7185	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7186	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7187	{
7188	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7189	if (RT_SUCCESS(rc))
7190	{
7191	pCtx->rip = u64Val;
7192	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7193	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7194	}
7195	return rc;
7196	}
7197	return VINF_SUCCESS;
7198	}
7199
7200
7201	/**
7202	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7203	*
7204	* @returns VBox status code.
7205	* @param pVCpu The cross context virtual CPU structure.
7206	* @param pVmcsInfo The VMCS info. object.
7207	*
7208	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7209	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7210	* instead!!!
7211	*/
7212	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7213	{
7214	uint32_t u32Val;
7215	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7216	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7217	{
7218	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7219	if (RT_SUCCESS(rc))
7220	{
7221	pCtx->eflags.u32 = u32Val;
7222
7223	/* Restore eflags for real-on-v86-mode hack. */
7224	if (pVmcsInfo->RealMode.fRealOnV86Active)
7225	{
7226	pCtx->eflags.Bits.u1VM = 0;
7227	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7228	}
7229	}
7230	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7231	return rc;
7232	}
7233	return VINF_SUCCESS;
7234	}
7235
7236
7237	/**
7238	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7239	* context.
7240	*
7241	* @returns VBox status code.
7242	* @param pVCpu The cross context virtual CPU structure.
7243	* @param pVmcsInfo The VMCS info. object.
7244	*
7245	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7246	* do not log!
7247	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7248	* instead!!!
7249	*/
7250	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7251	{
7252	uint32_t u32Val;
7253	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7254	if (RT_SUCCESS(rc))
7255	{
7256	if (!u32Val)
7257	{
7258	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7259	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7260
7261	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7262	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7263	}
7264	else
7265	{
7266	/*
7267	* We must import RIP here to set our EM interrupt-inhibited state.
7268	* We also import RFLAGS as our code that evaluates pending interrupts
7269	* before VM-entry requires it.
7270	*/
7271	rc = hmR0VmxImportGuestRip(pVCpu);
7272	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7273	if (RT_SUCCESS(rc))
7274	{
7275	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7276	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7277	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7278	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7279
7280	if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
7281	{
7282	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7283	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
7284	}
7285	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7286	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7287	}
7288	}
7289	}
7290	return rc;
7291	}
7292
7293
7294	/**
7295	* Worker for VMXR0ImportStateOnDemand.
7296	*
7297	* @returns VBox status code.
7298	* @param pVCpu The cross context virtual CPU structure.
7299	* @param pVmcsInfo The VMCS info. object.
7300	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7301	*/
7302	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7303	{
7304	#define VMXLOCAL_BREAK_RC(a_rc) \
7305	if (RT_SUCCESS(a_rc)) \
7306	{ } \
7307	else \
7308	break
7309
7310	int rc = VINF_SUCCESS;
7311	PVM pVM = pVCpu->CTX_SUFF(pVM);
7312	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7313	uint64_t u64Val;
7314	uint32_t u32Val;
7315
7316	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
7317
7318	/*
7319	* We disable interrupts to make the updating of the state and in particular
7320	* the fExtrn modification atomic wrt to preemption hooks.
7321	*/
7322	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7323
7324	fWhat &= pCtx->fExtrn;
7325	if (fWhat)
7326	{
7327	do
7328	{
7329	if (fWhat & CPUMCTX_EXTRN_RIP)
7330	{
7331	rc = hmR0VmxImportGuestRip(pVCpu);
7332	VMXLOCAL_BREAK_RC(rc);
7333	}
7334
7335	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7336	{
7337	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7338	VMXLOCAL_BREAK_RC(rc);
7339	}
7340
7341	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7342	{
7343	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7344	VMXLOCAL_BREAK_RC(rc);
7345	}
7346
7347	if (fWhat & CPUMCTX_EXTRN_RSP)
7348	{
7349	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
7350	VMXLOCAL_BREAK_RC(rc);
7351	pCtx->rsp = u64Val;
7352	}
7353
7354	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7355	{
7356	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7357	if (fWhat & CPUMCTX_EXTRN_CS)
7358	{
7359	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7360	rc \|= hmR0VmxImportGuestRip(pVCpu);
7361	if (fRealOnV86Active)
7362	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7363	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7364	}
7365	if (fWhat & CPUMCTX_EXTRN_SS)
7366	{
7367	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7368	if (fRealOnV86Active)
7369	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7370	}
7371	if (fWhat & CPUMCTX_EXTRN_DS)
7372	{
7373	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7374	if (fRealOnV86Active)
7375	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7376	}
7377	if (fWhat & CPUMCTX_EXTRN_ES)
7378	{
7379	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7380	if (fRealOnV86Active)
7381	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7382	}
7383	if (fWhat & CPUMCTX_EXTRN_FS)
7384	{
7385	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7386	if (fRealOnV86Active)
7387	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7388	}
7389	if (fWhat & CPUMCTX_EXTRN_GS)
7390	{
7391	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7392	if (fRealOnV86Active)
7393	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7394	}
7395	VMXLOCAL_BREAK_RC(rc);
7396	}
7397
7398	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7399	{
7400	if (fWhat & CPUMCTX_EXTRN_LDTR)
7401	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
7402
7403	if (fWhat & CPUMCTX_EXTRN_GDTR)
7404	{
7405	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
7406	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
7407	pCtx->gdtr.pGdt = u64Val;
7408	pCtx->gdtr.cbGdt = u32Val;
7409	}
7410
7411	/* Guest IDTR. */
7412	if (fWhat & CPUMCTX_EXTRN_IDTR)
7413	{
7414	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
7415	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
7416	pCtx->idtr.pIdt = u64Val;
7417	pCtx->idtr.cbIdt = u32Val;
7418	}
7419
7420	/* Guest TR. */
7421	if (fWhat & CPUMCTX_EXTRN_TR)
7422	{
7423	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7424	don't need to import that one. */
7425	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7426	rc \|= hmR0VmxImportGuestTr(pVCpu);
7427	}
7428	VMXLOCAL_BREAK_RC(rc);
7429	}
7430
7431	if (fWhat & CPUMCTX_EXTRN_DR7)
7432	{
7433	if (!pVCpu->hm.s.fUsingHyperDR7)
7434	{
7435	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
7436	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
7437	VMXLOCAL_BREAK_RC(rc);
7438	pCtx->dr[7] = u32Val;
7439	}
7440	}
7441
7442	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7443	{
7444	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
7445	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
7446	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
7447	pCtx->SysEnter.cs = u32Val;
7448	VMXLOCAL_BREAK_RC(rc);
7449	}
7450
7451	#if HC_ARCH_BITS == 64
7452	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7453	{
7454	if ( pVM->hm.s.fAllow64BitGuests
7455	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7456	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7457	}
7458
7459	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7460	{
7461	if ( pVM->hm.s.fAllow64BitGuests
7462	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7463	{
7464	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7465	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7466	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7467	}
7468	}
7469	#endif
7470
7471	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7472	#if HC_ARCH_BITS == 32
7473	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
7474	#endif
7475	)
7476	{
7477	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7478	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7479	Assert(pMsrs);
7480	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
7481	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
7482	for (uint32_t i = 0; i < cMsrs; i++)
7483	{
7484	uint32_t const idMsr = pMsrs[i].u32Msr;
7485	switch (idMsr)
7486	{
7487	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
7488	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
7489	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7490	#if HC_ARCH_BITS == 32
7491	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsrs[i].u64Value; break;
7492	case MSR_K6_STAR: pCtx->msrSTAR = pMsrs[i].u64Value; break;
7493	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsrs[i].u64Value; break;
7494	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsrs[i].u64Value; break;
7495	#endif
7496	default:
7497	{
7498	pCtx->fExtrn = 0;
7499	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
7500	ASMSetFlags(fEFlags);
7501	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
7502	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7503	}
7504	}
7505	}
7506	}
7507
7508	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7509	{
7510	uint64_t u64Shadow;
7511	if (fWhat & CPUMCTX_EXTRN_CR0)
7512	{
7513	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7514	* remove when we drop 32-bit host w/ 64-bit host support, see
7515	* @bugref{9180#c39}. */
7516	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7517	#if HC_ARCH_BITS == 32
7518	uint32_t u32Shadow;
7519	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
7520	u64Shadow = u32Shadow;
7521	#else
7522	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
7523	#endif
7524	VMXLOCAL_BREAK_RC(rc);
7525	u64Val = u32Val;
7526	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7527	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7528	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7529	CPUMSetGuestCR0(pVCpu, u64Val);
7530	VMMRZCallRing3Enable(pVCpu);
7531	}
7532
7533	if (fWhat & CPUMCTX_EXTRN_CR4)
7534	{
7535	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7536	* remove when we drop 32-bit host w/ 64-bit host support, see
7537	* @bugref{9180#c39}. */
7538	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7539	#if HC_ARCH_BITS == 32
7540	uint32_t u32Shadow;
7541	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
7542	u64Shadow = u32Shadow;
7543	#else
7544	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
7545	#endif
7546	VMXLOCAL_BREAK_RC(rc);
7547	u64Val = u32Val;
7548	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
7549	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7550	pCtx->cr4 = u64Val;
7551	}
7552
7553	if (fWhat & CPUMCTX_EXTRN_CR3)
7554	{
7555	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7556	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7557	\|\| ( pVM->hm.s.fNestedPaging
7558	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7559	{
7560	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
7561	VMXLOCAL_BREAK_RC(rc);
7562	if (pCtx->cr3 != u64Val)
7563	{
7564	pCtx->cr3 = u64Val;
7565	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7566	}
7567
7568	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7569	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7570	if (CPUMIsGuestInPAEModeEx(pCtx))
7571	{
7572	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
7573	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
7574	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
7575	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
7576	VMXLOCAL_BREAK_RC(rc);
7577	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7578	}
7579	}
7580	}
7581	}
7582	} while (0);
7583
7584	if (RT_SUCCESS(rc))
7585	{
7586	/* Update fExtrn. */
7587	pCtx->fExtrn &= ~fWhat;
7588
7589	/* If everything has been imported, clear the HM keeper bit. */
7590	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7591	{
7592	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7593	Assert(!pCtx->fExtrn);
7594	}
7595	}
7596	}
7597	else
7598	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7599
7600	ASMSetFlags(fEFlags);
7601
7602	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7603
7604	if (RT_SUCCESS(rc))
7605	{ /* likely */ }
7606	else
7607	return rc;
7608
7609	/*
7610	* Honor any pending CR3 updates.
7611	*
7612	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
7613	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7614	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7615	*
7616	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7617	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7618	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7619	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7620	*
7621	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7622	*/
7623	if (VMMRZCallRing3IsEnabled(pVCpu))
7624	{
7625	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7626	{
7627	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7628	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7629	}
7630
7631	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7632	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7633
7634	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7635	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7636	}
7637
7638	return VINF_SUCCESS;
7639	#undef VMXLOCAL_BREAK_RC
7640	}
7641
7642
7643	/**
7644	* Saves the guest state from the VMCS into the guest-CPU context.
7645	*
7646	* @returns VBox status code.
7647	* @param pVCpu The cross context virtual CPU structure.
7648	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7649	*/
7650	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
7651	{
7652	PCVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7653	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7654	}
7655
7656
7657	/**
7658	* Check per-VM and per-VCPU force flag actions that require us to go back to
7659	* ring-3 for one reason or another.
7660	*
7661	* @returns Strict VBox status code (i.e. informational status codes too)
7662	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7663	* ring-3.
7664	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7665	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7666	* interrupts)
7667	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7668	* all EMTs to be in ring-3.
7669	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7670	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7671	* to the EM loop.
7672	*
7673	* @param pVCpu The cross context virtual CPU structure.
7674	* @param fStepping Whether we are single-stepping the guest using the
7675	* hypervisor debugger.
7676	*/
7677	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
7678	{
7679	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7680
7681	/*
7682	* Update pending interrupts into the APIC's IRR.
7683	*/
7684	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7685	APICUpdatePendingInterrupts(pVCpu);
7686
7687	/*
7688	* Anything pending? Should be more likely than not if we're doing a good job.
7689	*/
7690	PVM pVM = pVCpu->CTX_SUFF(pVM);
7691	if ( !fStepping
7692	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7693	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7694	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7695	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7696	return VINF_SUCCESS;
7697
7698	/* Pending PGM C3 sync. */
7699	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7700	{
7701	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7702	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7703	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7704	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7705	if (rcStrict2 != VINF_SUCCESS)
7706	{
7707	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7708	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7709	return rcStrict2;
7710	}
7711	}
7712
7713	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7714	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7715	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7716	{
7717	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7718	int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7719	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7720	return rc2;
7721	}
7722
7723	/* Pending VM request packets, such as hardware interrupts. */
7724	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7725	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7726	{
7727	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7728	return VINF_EM_PENDING_REQUEST;
7729	}
7730
7731	/* Pending PGM pool flushes. */
7732	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7733	{
7734	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7735	return VINF_PGM_POOL_FLUSH_PENDING;
7736	}
7737
7738	/* Pending DMA requests. */
7739	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7740	{
7741	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7742	return VINF_EM_RAW_TO_R3;
7743	}
7744
7745	return VINF_SUCCESS;
7746	}
7747
7748
7749	/**
7750	* Converts any TRPM trap into a pending HM event. This is typically used when
7751	* entering from ring-3 (not longjmp returns).
7752	*
7753	* @param pVCpu The cross context virtual CPU structure.
7754	*/
7755	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7756	{
7757	Assert(TRPMHasTrap(pVCpu));
7758	Assert(!pVCpu->hm.s.Event.fPending);
7759
7760	uint8_t uVector;
7761	TRPMEVENT enmTrpmEvent;
7762	RTGCUINT uErrCode;
7763	RTGCUINTPTR GCPtrFaultAddress;
7764	uint8_t cbInstr;
7765
7766	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7767	AssertRC(rc);
7768
7769	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7770	uint32_t u32IntInfo = uVector \| VMX_EXIT_INT_INFO_VALID;
7771	if (enmTrpmEvent == TRPM_TRAP)
7772	{
7773	/** @todo r=ramshankar: TRPM currently offers no way to determine a \#DB that was
7774	* generated using INT1 (ICEBP). */
7775	switch (uVector)
7776	{
7777	case X86_XCPT_NMI:
7778	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7779	break;
7780
7781	case X86_XCPT_BP:
7782	case X86_XCPT_OF:
7783	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7784	break;
7785
7786	case X86_XCPT_PF:
7787	case X86_XCPT_DF:
7788	case X86_XCPT_TS:
7789	case X86_XCPT_NP:
7790	case X86_XCPT_SS:
7791	case X86_XCPT_GP:
7792	case X86_XCPT_AC:
7793	u32IntInfo \|= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7794	RT_FALL_THRU();
7795	default:
7796	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7797	break;
7798	}
7799	}
7800	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7801	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7802	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7803	{
7804	switch (uVector)
7805	{
7806	case X86_XCPT_BP:
7807	case X86_XCPT_OF:
7808	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7809	break;
7810
7811	default:
7812	Assert(uVector == X86_XCPT_DB);
7813	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7814	break;
7815	}
7816	}
7817	else
7818	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7819
7820	rc = TRPMResetTrap(pVCpu);
7821	AssertRC(rc);
7822	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7823	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7824
7825	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7826	}
7827
7828
7829	/**
7830	* Converts the pending HM event into a TRPM trap.
7831	*
7832	* @param pVCpu The cross context virtual CPU structure.
7833	*/
7834	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7835	{
7836	Assert(pVCpu->hm.s.Event.fPending);
7837
7838	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7839	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7840	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
7841	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7842
7843	/* If a trap was already pending, we did something wrong! */
7844	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7845
7846	/** @todo Use HMVmxEventToTrpmEventType() later. */
7847	TRPMEVENT enmTrapType;
7848	switch (uVectorType)
7849	{
7850	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7851	enmTrapType = TRPM_HARDWARE_INT;
7852	break;
7853
7854	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7855	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7856	enmTrapType = TRPM_TRAP;
7857	break;
7858
7859	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT: /* #DB (INT1/ICEBP). */
7860	Assert(uVector == X86_XCPT_DB);
7861	enmTrapType = TRPM_SOFTWARE_INT;
7862	break;
7863
7864	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP (INT3) and #OF (INTO) */
7865	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7866	enmTrapType = TRPM_SOFTWARE_INT;
7867	break;
7868
7869	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7870	enmTrapType = TRPM_SOFTWARE_INT;
7871	break;
7872
7873	default:
7874	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7875	enmTrapType = TRPM_32BIT_HACK;
7876	break;
7877	}
7878
7879	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7880
7881	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7882	AssertRC(rc);
7883
7884	if (fErrorCodeValid)
7885	TRPMSetErrorCode(pVCpu, uErrorCode);
7886
7887	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7888	&& uVector == X86_XCPT_PF)
7889	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7890	else if (enmTrapType == TRPM_SOFTWARE_INT)
7891	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7892
7893	/* We're now done converting the pending event. */
7894	pVCpu->hm.s.Event.fPending = false;
7895	}
7896
7897
7898	/**
7899	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7900	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7901	*
7902	* @param pVCpu The cross context virtual CPU structure.
7903	* @param pVmcsInfo The VMCS info. object.
7904	*/
7905	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
7906	{
7907	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7908	{
7909	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7910	{
7911	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7912	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7913	AssertRC(rc);
7914	}
7915	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
7916	}
7917
7918
7919	/**
7920	* Clears the interrupt-window exiting control in the VMCS.
7921	*
7922	* @param pVmcsInfo The VMCS info. object.
7923	*/
7924	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7925	{
7926	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7927	{
7928	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7929	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7930	}
7931	return VINF_SUCCESS;
7932	}
7933
7934
7935	/**
7936	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7937	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7938	*
7939	* @param pVCpu The cross context virtual CPU structure.
7940	* @param pVmcsInfo The VMCS info. object.
7941	*/
7942	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
7943	{
7944	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7945	{
7946	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7947	{
7948	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7949	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7950	AssertRC(rc);
7951	Log4Func(("Setup NMI-window exiting\n"));
7952	}
7953	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7954	}
7955
7956
7957	/**
7958	* Clears the NMI-window exiting control in the VMCS.
7959	*
7960	* @param pVmcsInfo The VMCS info. object.
7961	*/
7962	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7963	{
7964	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7965	{
7966	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7967	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7968	}
7969	return VINF_SUCCESS;
7970	}
7971
7972
7973	/**
7974	* Does the necessary state syncing before returning to ring-3 for any reason
7975	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7976	*
7977	* @returns VBox status code.
7978	* @param pVCpu The cross context virtual CPU structure.
7979	* @param fImportState Whether to import the guest state from the VMCS back
7980	* to the guest-CPU context.
7981	*
7982	* @remarks No-long-jmp zone!!!
7983	*/
7984	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
7985	{
7986	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7987	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7988
7989	RTCPUID idCpu = RTMpCpuId();
7990	Log4Func(("HostCpuId=%u\n", idCpu));
7991
7992	/*
7993	* !!! IMPORTANT !!!
7994	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7995	*/
7996
7997	/* Save the guest state if necessary. */
7998	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7999	if (fImportState)
8000	{
8001	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8002	AssertRCReturn(rc, rc);
8003	}
8004
8005	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8006	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8007	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8008
8009	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8010	#ifdef VBOX_STRICT
8011	if (CPUMIsHyperDebugStateActive(pVCpu))
8012	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8013	#endif
8014	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8015	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8016	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8017
8018	#if HC_ARCH_BITS == 64
8019	/* Restore host-state bits that VT-x only restores partially. */
8020	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8021	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8022	{
8023	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8024	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8025	}
8026	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8027	#endif
8028
8029	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8030	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8031	{
8032	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8033	if (!fImportState)
8034	{
8035	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8036	AssertRCReturn(rc, rc);
8037	}
8038	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8039	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8040	}
8041	else
8042	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8043
8044	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8045	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8046
8047	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8048	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8049	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8050	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8051	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8052	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8053	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8054	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8055	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8056
8057	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8058
8059	/** @todo This partially defeats the purpose of having preemption hooks.
8060	* The problem is, deregistering the hooks should be moved to a place that
8061	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8062	* context.
8063	*/
8064	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8065	AssertRCReturn(rc, rc);
8066
8067	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8068	NOREF(idCpu);
8069	return VINF_SUCCESS;
8070	}
8071
8072
8073	/**
8074	* Leaves the VT-x session.
8075	*
8076	* @returns VBox status code.
8077	* @param pVCpu The cross context virtual CPU structure.
8078	*
8079	* @remarks No-long-jmp zone!!!
8080	*/
8081	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8082	{
8083	HM_DISABLE_PREEMPT(pVCpu);
8084	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8085	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8086	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8087
8088	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8089	and done this from the VMXR0ThreadCtxCallback(). */
8090	if (!pVCpu->hm.s.fLeaveDone)
8091	{
8092	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8093	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8094	pVCpu->hm.s.fLeaveDone = true;
8095	}
8096	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8097
8098	/*
8099	* !!! IMPORTANT !!!
8100	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8101	*/
8102
8103	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8104	/** @todo Deregistering here means we need to VMCLEAR always
8105	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8106	* for calling VMMR0ThreadCtxHookDisable here! */
8107	VMMR0ThreadCtxHookDisable(pVCpu);
8108
8109	/* Leave HM context. This takes care of local init (term). */
8110	int rc = HMR0LeaveCpu(pVCpu);
8111
8112	HM_RESTORE_PREEMPT();
8113	return rc;
8114	}
8115
8116
8117	/**
8118	* Does the necessary state syncing before doing a longjmp to ring-3.
8119	*
8120	* @returns VBox status code.
8121	* @param pVCpu The cross context virtual CPU structure.
8122	*
8123	* @remarks No-long-jmp zone!!!
8124	*/
8125	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8126	{
8127	return hmR0VmxLeaveSession(pVCpu);
8128	}
8129
8130
8131	/**
8132	* Take necessary actions before going back to ring-3.
8133	*
8134	* An action requires us to go back to ring-3. This function does the necessary
8135	* steps before we can safely return to ring-3. This is not the same as longjmps
8136	* to ring-3, this is voluntary and prepares the guest so it may continue
8137	* executing outside HM (recompiler/IEM).
8138	*
8139	* @returns VBox status code.
8140	* @param pVCpu The cross context virtual CPU structure.
8141	* @param rcExit The reason for exiting to ring-3. Can be
8142	* VINF_VMM_UNKNOWN_RING3_CALL.
8143	*/
8144	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8145	{
8146	Assert(pVCpu);
8147	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8148
8149	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8150	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8151	{
8152	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8153	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8154	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8155	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8156	}
8157
8158	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8159	VMMRZCallRing3Disable(pVCpu);
8160	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8161
8162	/*
8163	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8164	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8165	*
8166	* This is because execution may continue from ring-3 and we would need to inject
8167	* the event from there (hence place it back in TRPM).
8168	*/
8169	if (pVCpu->hm.s.Event.fPending)
8170	{
8171	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8172	Assert(!pVCpu->hm.s.Event.fPending);
8173
8174	/* Clear the events from the VMCS. */
8175	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8176	AssertRCReturn(rc, rc);
8177	}
8178	#ifdef VBOX_STRICT
8179	else
8180	{
8181	/*
8182	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8183	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8184	* occasionally, see @bugref{9180#c42}.
8185	*/
8186	uint32_t uEntryIntInfo;
8187	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8188	AssertRC(rc);
8189	Assert(!VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8190	}
8191	#endif
8192
8193	/*
8194	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8195	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8196	* (e.g. TPR below threshold).
8197	*/
8198	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8199	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8200	AssertRCReturn(rc, rc);
8201
8202	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8203	and if we're injecting an event we should have a TRPM trap pending. */
8204	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8205	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8206	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8207	#endif
8208
8209	/* Save guest state and restore host state bits. */
8210	rc = hmR0VmxLeaveSession(pVCpu);
8211	AssertRCReturn(rc, rc);
8212	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8213
8214	/* Thread-context hooks are unregistered at this point!!! */
8215
8216	/* Sync recompiler state. */
8217	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8218	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8219	\| CPUM_CHANGED_LDTR
8220	\| CPUM_CHANGED_GDTR
8221	\| CPUM_CHANGED_IDTR
8222	\| CPUM_CHANGED_TR
8223	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8224	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8225	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8226	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8227
8228	Assert(!pVCpu->hm.s.fClearTrapFlag);
8229
8230	/* Update the exit-to-ring 3 reason. */
8231	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8232
8233	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8234	if ( rcExit != VINF_EM_RAW_INTERRUPT
8235	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8236	{
8237	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8238	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8239	}
8240
8241	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8242
8243	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
8244	VMMRZCallRing3RemoveNotification(pVCpu);
8245	VMMRZCallRing3Enable(pVCpu);
8246
8247	return rc;
8248	}
8249
8250
8251	/**
8252	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8253	* longjump to ring-3 and possibly get preempted.
8254	*
8255	* @returns VBox status code.
8256	* @param pVCpu The cross context virtual CPU structure.
8257	* @param enmOperation The operation causing the ring-3 longjump.
8258	* @param pvUser User argument, currently unused, NULL.
8259	*/
8260	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
8261	{
8262	RT_NOREF(pvUser);
8263	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8264	{
8265	/*
8266	* !!! IMPORTANT !!!
8267	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8268	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8269	*/
8270	VMMRZCallRing3RemoveNotification(pVCpu);
8271	VMMRZCallRing3Disable(pVCpu);
8272	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
8273	RTThreadPreemptDisable(&PreemptState);
8274
8275	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8276	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8277	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8278	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8279
8280	#if HC_ARCH_BITS == 64
8281	/* Restore host-state bits that VT-x only restores partially. */
8282	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8283	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8284	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8285	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8286	#endif
8287
8288	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8289	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8290	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8291
8292	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8293	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8294	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8295
8296	/* Clear the current VMCS data back to memory. */
8297	hmR0VmxClearVmcs(pVmcsInfo);
8298
8299	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8300	VMMR0ThreadCtxHookDisable(pVCpu);
8301	HMR0LeaveCpu(pVCpu);
8302	RTThreadPreemptRestore(&PreemptState);
8303	return VINF_SUCCESS;
8304	}
8305
8306	Assert(pVCpu);
8307	Assert(pvUser);
8308	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8309	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8310
8311	VMMRZCallRing3Disable(pVCpu);
8312	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8313
8314	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8315
8316	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8317	AssertRCReturn(rc, rc);
8318
8319	VMMRZCallRing3Enable(pVCpu);
8320	return VINF_SUCCESS;
8321	}
8322
8323
8324	/**
8325	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8326	* stack.
8327	*
8328	* @returns Strict VBox status code (i.e. informational status codes too).
8329	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8330	* @param pVCpu The cross context virtual CPU structure.
8331	* @param uValue The value to push to the guest stack.
8332	*/
8333	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
8334	{
8335	/*
8336	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8337	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8338	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8339	*/
8340	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8341	if (pCtx->sp == 1)
8342	return VINF_EM_RESET;
8343	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8344	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8345	AssertRC(rc);
8346	return rc;
8347	}
8348
8349
8350	/**
8351	* Injects an event into the guest upon VM-entry by updating the relevant fields
8352	* in the VM-entry area in the VMCS.
8353	*
8354	* @returns Strict VBox status code (i.e. informational status codes too).
8355	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8356	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8357	*
8358	* @param pVCpu The cross context virtual CPU structure.
8359	* @param pVmxTransient The VMX-transient structure.
8360	* @param pEvent The event being injected.
8361	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state.
8362	* This will be updated if necessary. This cannot not
8363	* be NULL.
8364	* @param fStepping Whether we're single-stepping guest execution and
8365	* should return VINF_EM_DBG_STEPPED if the event is
8366	* injected directly (registers modified by us, not by
8367	* hardware on VM-entry).
8368	*/
8369	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8370	uint32_t *pfIntrState)
8371	{
8372	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8373	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8374	Assert(pfIntrState);
8375
8376	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8377	uint32_t u32IntInfo = pEvent->u64IntInfo;
8378	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8379	uint32_t const cbInstr = pEvent->cbInstr;
8380	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8381	uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8382	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8383
8384	#ifdef VBOX_STRICT
8385	/*
8386	* Validate the error-code-valid bit for hardware exceptions.
8387	* No error codes for exceptions in real-mode.
8388	*
8389	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8390	*/
8391	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8392	&& !CPUMIsGuestInRealModeEx(pCtx))
8393	{
8394	switch (uVector)
8395	{
8396	case X86_XCPT_PF:
8397	case X86_XCPT_DF:
8398	case X86_XCPT_TS:
8399	case X86_XCPT_NP:
8400	case X86_XCPT_SS:
8401	case X86_XCPT_GP:
8402	case X86_XCPT_AC:
8403	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8404	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8405	RT_FALL_THRU();
8406	default:
8407	break;
8408	}
8409	}
8410
8411	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8412	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8413	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8414	#endif
8415
8416	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8417
8418	/*
8419	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8420	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8421	* interrupt handler in the (real-mode) guest.
8422	*
8423	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8424	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8425	*/
8426	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8427	{
8428	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8429	{
8430	/*
8431	* For CPUs with unrestricted guest execution enabled and with the guest
8432	* in real-mode, we must not set the deliver-error-code bit.
8433	*
8434	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8435	*/
8436	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8437	}
8438	else
8439	{
8440	PVM pVM = pVCpu->CTX_SUFF(pVM);
8441	Assert(PDMVmmDevHeapIsEnabled(pVM));
8442	Assert(pVM->hm.s.vmx.pRealModeTSS);
8443	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8444
8445	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8446	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8447	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8448	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8449	AssertRCReturn(rc2, rc2);
8450
8451	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8452	size_t const cbIdtEntry = sizeof(X86IDTR16);
8453	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8454	{
8455	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8456	if (uVector == X86_XCPT_DF)
8457	return VINF_EM_RESET;
8458
8459	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8460	No error codes for exceptions in real-mode. */
8461	if (uVector == X86_XCPT_GP)
8462	{
8463	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8464	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8465	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8466	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8467	HMEVENT EventXcptDf;
8468	RT_ZERO(EventXcptDf);
8469	EventXcptDf.u64IntInfo = uXcptDfInfo;
8470	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8471	}
8472
8473	/*
8474	* If we're injecting an event with no valid IDT entry, inject a #GP.
8475	* No error codes for exceptions in real-mode.
8476	*
8477	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8478	*/
8479	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8480	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8481	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8482	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8483	HMEVENT EventXcptGp;
8484	RT_ZERO(EventXcptGp);
8485	EventXcptGp.u64IntInfo = uXcptGpInfo;
8486	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8487	}
8488
8489	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8490	uint16_t uGuestIp = pCtx->ip;
8491	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8492	{
8493	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8494	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8495	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8496	}
8497	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8498	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8499
8500	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8501	X86IDTR16 IdtEntry;
8502	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8503	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8504	AssertRCReturn(rc2, rc2);
8505
8506	/* Construct the stack frame for the interrupt/exception handler. */
8507	VBOXSTRICTRC rcStrict;
8508	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8509	if (rcStrict == VINF_SUCCESS)
8510	{
8511	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8512	if (rcStrict == VINF_SUCCESS)
8513	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8514	}
8515
8516	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8517	if (rcStrict == VINF_SUCCESS)
8518	{
8519	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8520	pCtx->rip = IdtEntry.offSel;
8521	pCtx->cs.Sel = IdtEntry.uSel;
8522	pCtx->cs.ValidSel = IdtEntry.uSel;
8523	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8524	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8525	&& uVector == X86_XCPT_PF)
8526	pCtx->cr2 = GCPtrFault;
8527
8528	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8529	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8530	\| HM_CHANGED_GUEST_RSP);
8531
8532	/*
8533	* If we delivered a hardware exception (other than an NMI) and if there was
8534	* block-by-STI in effect, we should clear it.
8535	*/
8536	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8537	{
8538	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8539	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8540	Log4Func(("Clearing inhibition due to STI\n"));
8541	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8542	}
8543
8544	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8545	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8546
8547	/*
8548	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8549	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8550	*/
8551	pVCpu->hm.s.Event.fPending = false;
8552
8553	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8554	if (fStepping)
8555	rcStrict = VINF_EM_DBG_STEPPED;
8556	}
8557	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8558	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8559	return rcStrict;
8560	}
8561	}
8562
8563	/*
8564	* Validate.
8565	*/
8566	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8567	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8568
8569	/*
8570	* Inject the event into the VMCS.
8571	*/
8572	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8573	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8574	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8575	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8576	AssertRCReturn(rc, rc);
8577
8578	/*
8579	* Update guest CR2 if this is a page-fault.
8580	*/
8581	if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8582	&& uVector == X86_XCPT_PF)
8583	pCtx->cr2 = GCPtrFault;
8584
8585	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8586	return VINF_SUCCESS;
8587	}
8588
8589
8590	/**
8591	* Evaluates the event to be delivered to the guest and sets it as the pending
8592	* event.
8593	*
8594	* @returns Strict VBox status code (i.e. informational status codes too).
8595	* @param pVCpu The cross context virtual CPU structure.
8596	* @param pVmxTransient The VMX-transient structure.
8597	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8598	*/
8599	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8600	{
8601	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8602	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8603
8604	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
8605	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmcsInfo);
8606	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8607	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8608	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8609
8610	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8611	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8612	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8613	Assert(!TRPMHasTrap(pVCpu));
8614	Assert(pfIntrState);
8615
8616	*pfIntrState = fIntrState;
8617
8618	/*
8619	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
8620	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
8621	*/
8622	/** @todo SMI. SMIs take priority over NMIs. */
8623	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
8624	{
8625	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8626	if ( !pVCpu->hm.s.Event.fPending
8627	&& !fBlockNmi
8628	&& !fBlockSti
8629	&& !fBlockMovSS)
8630	{
8631	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8632	if ( pVmxTransient->fIsNestedGuest
8633	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8634	return IEMExecVmxVmexitNmi(pVCpu);
8635	#endif
8636	hmR0VmxSetPendingXcptNmi(pVCpu);
8637	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8638	Log4Func(("Pending NMI\n"));
8639	}
8640	else
8641	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8642	}
8643	/*
8644	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
8645	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
8646	*/
8647	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8648	&& !pVCpu->hm.s.fSingleInstruction)
8649	{
8650	Assert(!DBGFIsStepping(pVCpu));
8651	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8652	AssertRCReturn(rc, rc);
8653	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8654	if ( !pVCpu->hm.s.Event.fPending
8655	&& !fBlockInt
8656	&& !fBlockSti
8657	&& !fBlockMovSS)
8658	{
8659	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8660	if ( pVmxTransient->fIsNestedGuest
8661	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
8662	{
8663	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector /, true / fIntPending */);
8664	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8665	return rcStrict;
8666	}
8667	#endif
8668	uint8_t u8Interrupt;
8669	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8670	if (RT_SUCCESS(rc))
8671	{
8672	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8673	if ( pVmxTransient->fIsNestedGuest
8674	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8675	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8676	{
8677	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8678	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8679	return rcStrict;
8680	}
8681	#endif
8682	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8683	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
8684	}
8685	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8686	{
8687	if ( !pVmxTransient->fIsNestedGuest
8688	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8689	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
8690	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8691
8692	/*
8693	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8694	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8695	* need to re-set this force-flag here.
8696	*/
8697	}
8698	else
8699	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8700	}
8701	else
8702	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8703	}
8704
8705	return VINF_SUCCESS;
8706	}
8707
8708
8709	/**
8710	* Injects any pending events into the guest if the guest is in a state to
8711	* receive them.
8712	*
8713	* @returns Strict VBox status code (i.e. informational status codes too).
8714	* @param pVCpu The cross context virtual CPU structure.
8715	* @param pVmxTransient The VMX-transient structure.
8716	* @param fIntrState The VT-x guest-interruptibility state.
8717	* @param fStepping Whether we are single-stepping the guest using the
8718	* hypervisor debugger and should return
8719	* VINF_EM_DBG_STEPPED if the event was dispatched
8720	* directly.
8721	*/
8722	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
8723	{
8724	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8725	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8726
8727	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8728	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8729
8730	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8731	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8732	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8733	Assert(!TRPMHasTrap(pVCpu));
8734
8735	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8736	if (pVCpu->hm.s.Event.fPending)
8737	{
8738	/*
8739	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8740	* pending even while injecting an event and in this case, we want a VM-exit as soon as
8741	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
8742	*
8743	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8744	*/
8745	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8746	#ifdef VBOX_STRICT
8747	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8748	{
8749	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
8750	Assert(!fBlockInt);
8751	Assert(!fBlockSti);
8752	Assert(!fBlockMovSS);
8753	}
8754	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
8755	{
8756	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8757	Assert(!fBlockSti);
8758	Assert(!fBlockMovSS);
8759	Assert(!fBlockNmi);
8760	}
8761	#endif
8762	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8763	uIntType));
8764
8765	/*
8766	* Inject the event and get any changes to the guest-interruptibility state.
8767	*
8768	* The guest-interruptibility state may need to be updated if we inject the event
8769	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
8770	*/
8771	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
8772	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8773
8774	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8775	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8776	else
8777	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8778	}
8779
8780	/*
8781	* Update the guest-interruptibility state.
8782	*
8783	* This is required for the real-on-v86 software interrupt injection case above, as well as
8784	* updates to the guest state from ring-3 or IEM/REM.
8785	*/
8786	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
8787	AssertRCReturn(rc, rc);
8788
8789	/*
8790	* There's no need to clear the VM-entry interruption-information field here if we're not
8791	* injecting anything. VT-x clears the valid bit on every VM-exit.
8792	*
8793	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8794	*/
8795
8796	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8797	NOREF(fBlockMovSS); NOREF(fBlockSti);
8798	return rcStrict;
8799	}
8800
8801
8802	/**
8803	* Enters the VT-x session.
8804	*
8805	* @returns VBox status code.
8806	* @param pVCpu The cross context virtual CPU structure.
8807	*/
8808	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
8809	{
8810	AssertPtr(pVCpu);
8811	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8812	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8813
8814	LogFlowFunc(("pVCpu=%p\n", pVCpu));
8815	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8816	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8817
8818	#ifdef VBOX_STRICT
8819	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8820	RTCCUINTREG uHostCR4 = ASMGetCR4();
8821	if (!(uHostCR4 & X86_CR4_VMXE))
8822	{
8823	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8824	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8825	}
8826	#endif
8827
8828	/*
8829	* Load the appropriate VMCS as the current and active one.
8830	*/
8831	PVMXVMCSINFO pVmcsInfo;
8832	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
8833	if (!fInNestedGuestMode)
8834	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
8835	else
8836	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
8837	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
8838	if (RT_SUCCESS(rc))
8839	{
8840	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
8841	pVCpu->hm.s.fLeaveDone = false;
8842	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8843
8844	/*
8845	* Do the EMT scheduled L1D flush here if needed.
8846	*/
8847	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8848	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8849	}
8850	return rc;
8851	}
8852
8853
8854	/**
8855	* The thread-context callback (only on platforms which support it).
8856	*
8857	* @param enmEvent The thread-context event.
8858	* @param pVCpu The cross context virtual CPU structure.
8859	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8860	* @thread EMT(pVCpu)
8861	*/
8862	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8863	{
8864	NOREF(fGlobalInit);
8865
8866	switch (enmEvent)
8867	{
8868	case RTTHREADCTXEVENT_OUT:
8869	{
8870	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8871	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8872	VMCPU_ASSERT_EMT(pVCpu);
8873
8874	/* No longjmps (logger flushes, locks) in this fragile context. */
8875	VMMRZCallRing3Disable(pVCpu);
8876	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8877
8878	/* Restore host-state (FPU, debug etc.) */
8879	if (!pVCpu->hm.s.fLeaveDone)
8880	{
8881	/*
8882	* Do -not- import the guest-state here as we might already be in the middle of importing
8883	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8884	*/
8885	hmR0VmxLeave(pVCpu, false /* fImportState */);
8886	pVCpu->hm.s.fLeaveDone = true;
8887	}
8888
8889	/* Leave HM context, takes care of local init (term). */
8890	int rc = HMR0LeaveCpu(pVCpu);
8891	AssertRC(rc);
8892
8893	/* Restore longjmp state. */
8894	VMMRZCallRing3Enable(pVCpu);
8895	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8896	break;
8897	}
8898
8899	case RTTHREADCTXEVENT_IN:
8900	{
8901	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8902	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8903	VMCPU_ASSERT_EMT(pVCpu);
8904
8905	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8906	VMMRZCallRing3Disable(pVCpu);
8907	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8908
8909	/* Initialize the bare minimum state required for HM. This takes care of
8910	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8911	int rc = hmR0EnterCpu(pVCpu);
8912	AssertRC(rc);
8913	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8914	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8915
8916	/* Load the active VMCS as the current one. */
8917	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8918	rc = hmR0VmxLoadVmcs(pVmcsInfo);
8919	AssertRC(rc);
8920	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8921	pVCpu->hm.s.fLeaveDone = false;
8922
8923	/* Do the EMT scheduled L1D flush if needed. */
8924	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8925	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8926
8927	/* Restore longjmp state. */
8928	VMMRZCallRing3Enable(pVCpu);
8929	break;
8930	}
8931
8932	default:
8933	break;
8934	}
8935	}
8936
8937
8938	/**
8939	* Exports the host state into the VMCS host-state area.
8940	* Sets up the VM-exit MSR-load area.
8941	*
8942	* The CPU state will be loaded from these fields on every successful VM-exit.
8943	*
8944	* @returns VBox status code.
8945	* @param pVCpu The cross context virtual CPU structure.
8946	*
8947	* @remarks No-long-jump zone!!!
8948	*/
8949	static int hmR0VmxExportHostState(PVMCPU pVCpu)
8950	{
8951	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8952
8953	int rc = VINF_SUCCESS;
8954	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8955	{
8956	rc = hmR0VmxExportHostControlRegs();
8957	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8958
8959	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8960	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8961
8962	rc = hmR0VmxExportHostMsrs(pVCpu);
8963	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8964
8965	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8966	}
8967	return rc;
8968	}
8969
8970
8971	/**
8972	* Saves the host state in the VMCS host-state.
8973	*
8974	* @returns VBox status code.
8975	* @param pVCpu The cross context virtual CPU structure.
8976	*
8977	* @remarks No-long-jump zone!!!
8978	*/
8979	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
8980	{
8981	AssertPtr(pVCpu);
8982	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8983
8984	/*
8985	* Export the host state here while entering HM context.
8986	* When thread-context hooks are used, we might get preempted and have to re-save the host
8987	* state but most of the time we won't be, so do it here before we disable interrupts.
8988	*/
8989	return hmR0VmxExportHostState(pVCpu);
8990	}
8991
8992
8993	/**
8994	* Exports the guest state into the VMCS guest-state area.
8995	*
8996	* The will typically be done before VM-entry when the guest-CPU state and the
8997	* VMCS state may potentially be out of sync.
8998	*
8999	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9000	* VM-entry controls.
9001	* Sets up the appropriate VMX non-root function to execute guest code based on
9002	* the guest CPU mode.
9003	*
9004	* @returns VBox strict status code.
9005	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9006	* without unrestricted guest execution and the VMMDev is not presently
9007	* mapped (e.g. EFI32).
9008	*
9009	* @param pVCpu The cross context virtual CPU structure.
9010	* @param pVmxTransient The VMX-transient structure.
9011	*
9012	* @remarks No-long-jump zone!!!
9013	*/
9014	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9015	{
9016	AssertPtr(pVCpu);
9017	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9018	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9019
9020	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9021
9022	/*
9023	* Determine real-on-v86 mode.
9024	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9025	*/
9026	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9027	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9028	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9029	pVmcsInfo->RealMode. fRealOnV86Active = false;
9030	else
9031	{
9032	Assert(!pVmxTransient->fIsNestedGuest);
9033	pVmcsInfo->RealMode.fRealOnV86Active = true;
9034	}
9035
9036	/*
9037	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9038	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9039	*/
9040	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9041	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9042	* be a need to evaluate this everytime since I'm pretty sure we intercept
9043	* all guest paging mode changes. */
9044	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9045	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9046
9047	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9048	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9049
9050	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9051	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9052
9053	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9054	if (rcStrict == VINF_SUCCESS)
9055	{ /* likely */ }
9056	else
9057	{
9058	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9059	return rcStrict;
9060	}
9061
9062	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9063	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9064
9065	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9066	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9067
9068	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9069	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9070
9071	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9072	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9073
9074	rc = hmR0VmxExportGuestRip(pVCpu);
9075	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9076	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9077	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9078
9079	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9080	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9081	\| HM_CHANGED_GUEST_CR2
9082	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9083	\| HM_CHANGED_GUEST_X87
9084	\| HM_CHANGED_GUEST_SSE_AVX
9085	\| HM_CHANGED_GUEST_OTHER_XSAVE
9086	\| HM_CHANGED_GUEST_XCRx
9087	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9088	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9089	\| HM_CHANGED_GUEST_TSC_AUX
9090	\| HM_CHANGED_GUEST_OTHER_MSRS
9091	\| HM_CHANGED_GUEST_HWVIRT /* More accurate PLE handling someday? */
9092	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9093
9094	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9095	return rc;
9096	}
9097
9098
9099	/**
9100	* Exports the state shared between the host and guest into the VMCS.
9101	*
9102	* @param pVCpu The cross context virtual CPU structure.
9103	* @param pVmxTransient The VMX-transient structure.
9104	*
9105	* @remarks No-long-jump zone!!!
9106	*/
9107	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9108	{
9109	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9110	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9111
9112	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9113	{
9114	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9115	AssertRC(rc);
9116	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9117
9118	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9119	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9120	{
9121	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9122	AssertRC(rc);
9123	}
9124	}
9125
9126	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9127	{
9128	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9129	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9130	}
9131
9132	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9133	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9134	}
9135
9136
9137	/**
9138	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9139	*
9140	* @returns Strict VBox status code (i.e. informational status codes too).
9141	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9142	* without unrestricted guest execution and the VMMDev is not presently
9143	* mapped (e.g. EFI32).
9144	*
9145	* @param pVCpu The cross context virtual CPU structure.
9146	* @param pVmxTransient The VMX-transient structure.
9147	*
9148	* @remarks No-long-jump zone!!!
9149	*/
9150	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9151	{
9152	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9153	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9154	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9155
9156	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9157	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9158	#endif
9159
9160	/*
9161	* For many exits it's only RIP that changes and hence try to export it first
9162	* without going through a lot of change flag checks.
9163	*/
9164	VBOXSTRICTRC rcStrict;
9165	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9166	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9167	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9168	{
9169	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9170	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9171	{ /* likely */}
9172	else
9173	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9174	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9175	}
9176	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9177	{
9178	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9179	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9180	{ /* likely */}
9181	else
9182	{
9183	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9184	VBOXSTRICTRC_VAL(rcStrict)));
9185	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9186	return rcStrict;
9187	}
9188	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9189	}
9190	else
9191	rcStrict = VINF_SUCCESS;
9192
9193	#ifdef VBOX_STRICT
9194	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9195	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9196	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9197	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9198	("fCtxChanged=%#RX64\n", fCtxChanged));
9199	#endif
9200	return rcStrict;
9201	}
9202
9203
9204	/**
9205	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9206	* and update error record fields accordingly.
9207	*
9208	* @return VMX_IGS_* return codes.
9209	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9210	* wrong with the guest state.
9211	*
9212	* @param pVCpu The cross context virtual CPU structure.
9213	* @param pVmcsInfo The VMCS info. object.
9214	*
9215	* @remarks This function assumes our cache of the VMCS controls
9216	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9217	*/
9218	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
9219	{
9220	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9221	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9222	uError = (err); \
9223	break; \
9224	} else do { } while (0)
9225
9226	int rc;
9227	PVM pVM = pVCpu->CTX_SUFF(pVM);
9228	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9229	uint32_t uError = VMX_IGS_ERROR;
9230	uint32_t u32Val;
9231	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9232
9233	do
9234	{
9235	/*
9236	* CR0.
9237	*/
9238	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9239	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9240	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
9241	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9242	if (fUnrestrictedGuest)
9243	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
9244
9245	uint32_t u32GuestCr0;
9246	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
9247	AssertRCBreak(rc);
9248	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9249	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9250	if ( !fUnrestrictedGuest
9251	&& (u32GuestCr0 & X86_CR0_PG)
9252	&& !(u32GuestCr0 & X86_CR0_PE))
9253	{
9254	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9255	}
9256
9257	/*
9258	* CR4.
9259	*/
9260	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9261	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9262
9263	uint32_t u32GuestCr4;
9264	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
9265	AssertRCBreak(rc);
9266	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9267	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9268
9269	/*
9270	* IA32_DEBUGCTL MSR.
9271	*/
9272	uint64_t u64Val;
9273	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9274	AssertRCBreak(rc);
9275	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9276	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9277	{
9278	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9279	}
9280	uint64_t u64DebugCtlMsr = u64Val;
9281
9282	#ifdef VBOX_STRICT
9283	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9284	AssertRCBreak(rc);
9285	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9286	#endif
9287	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9288
9289	/*
9290	* RIP and RFLAGS.
9291	*/
9292	uint32_t u32Eflags;
9293	#if HC_ARCH_BITS == 64
9294	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9295	AssertRCBreak(rc);
9296	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9297	if ( !fLongModeGuest
9298	\|\| !pCtx->cs.Attr.n.u1Long)
9299	{
9300	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9301	}
9302	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9303	* must be identical if the "IA-32e mode guest" VM-entry
9304	* control is 1 and CS.L is 1. No check applies if the
9305	* CPU supports 64 linear-address bits. */
9306
9307	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9308	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9309	AssertRCBreak(rc);
9310	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9311	VMX_IGS_RFLAGS_RESERVED);
9312	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9313	u32Eflags = u64Val;
9314	#else
9315	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9316	AssertRCBreak(rc);
9317	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9318	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9319	#endif
9320
9321	if ( fLongModeGuest
9322	\|\| ( fUnrestrictedGuest
9323	&& !(u32GuestCr0 & X86_CR0_PE)))
9324	{
9325	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9326	}
9327
9328	uint32_t u32EntryInfo;
9329	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9330	AssertRCBreak(rc);
9331	if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9332	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9333	{
9334	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9335	}
9336
9337	/*
9338	* 64-bit checks.
9339	*/
9340	#if HC_ARCH_BITS == 64
9341	if (fLongModeGuest)
9342	{
9343	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9344	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9345	}
9346
9347	if ( !fLongModeGuest
9348	&& (u32GuestCr4 & X86_CR4_PCIDE))
9349	{
9350	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9351	}
9352
9353	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9354	* 51:32 beyond the processor's physical-address width are 0. */
9355
9356	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9357	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9358	{
9359	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9360	}
9361
9362	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9363	AssertRCBreak(rc);
9364	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9365
9366	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9367	AssertRCBreak(rc);
9368	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9369	#endif
9370
9371	/*
9372	* PERF_GLOBAL MSR.
9373	*/
9374	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9375	{
9376	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9377	AssertRCBreak(rc);
9378	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9379	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9380	}
9381
9382	/*
9383	* PAT MSR.
9384	*/
9385	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9386	{
9387	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9388	AssertRCBreak(rc);
9389	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9390	for (unsigned i = 0; i < 8; i++)
9391	{
9392	uint8_t u8Val = (u64Val & 0xff);
9393	if ( u8Val != 0 /* UC */
9394	&& u8Val != 1 /* WC */
9395	&& u8Val != 4 /* WT */
9396	&& u8Val != 5 /* WP */
9397	&& u8Val != 6 /* WB */
9398	&& u8Val != 7 /* UC- */)
9399	{
9400	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9401	}
9402	u64Val >>= 8;
9403	}
9404	}
9405
9406	/*
9407	* EFER MSR.
9408	*/
9409	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9410	{
9411	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9412	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9413	AssertRCBreak(rc);
9414	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9415	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9416	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9417	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9418	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9419	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9420	* iemVmxVmentryCheckGuestState(). */
9421	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9422	\|\| !(u32GuestCr0 & X86_CR0_PG)
9423	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9424	VMX_IGS_EFER_LMA_LME_MISMATCH);
9425	}
9426
9427	/*
9428	* Segment registers.
9429	*/
9430	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9431	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9432	if (!(u32Eflags & X86_EFL_VM))
9433	{
9434	/* CS */
9435	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9436	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9437	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9438	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9439	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9440	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9441	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9442	/* CS cannot be loaded with NULL in protected mode. */
9443	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9444	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9445	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9446	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9447	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9448	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9449	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9450	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9451	else
9452	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9453
9454	/* SS */
9455	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9456	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9457	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9458	if ( !(pCtx->cr0 & X86_CR0_PE)
9459	\|\| pCtx->cs.Attr.n.u4Type == 3)
9460	{
9461	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9462	}
9463	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9464	{
9465	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9466	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9467	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9468	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9469	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9470	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9471	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9472	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9473	}
9474
9475	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9476	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9477	{
9478	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9479	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9480	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9481	\|\| pCtx->ds.Attr.n.u4Type > 11
9482	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9483	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9484	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9485	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9486	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9487	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9488	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9489	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9490	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9491	}
9492	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9493	{
9494	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9495	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9496	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9497	\|\| pCtx->es.Attr.n.u4Type > 11
9498	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9499	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9500	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9501	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9502	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9503	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9504	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9505	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9506	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9507	}
9508	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9509	{
9510	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9511	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9512	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9513	\|\| pCtx->fs.Attr.n.u4Type > 11
9514	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9515	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9516	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9517	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9518	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9519	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9520	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9521	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9522	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9523	}
9524	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9525	{
9526	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9527	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9528	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9529	\|\| pCtx->gs.Attr.n.u4Type > 11
9530	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9531	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9532	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9533	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9534	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9535	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9536	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9537	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9538	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9539	}
9540	/* 64-bit capable CPUs. */
9541	#if HC_ARCH_BITS == 64
9542	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9543	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9544	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9545	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9546	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9547	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9548	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9549	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9550	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9551	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9552	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9553	#endif
9554	}
9555	else
9556	{
9557	/* V86 mode checks. */
9558	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9559	if (pVmcsInfo->RealMode.fRealOnV86Active)
9560	{
9561	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9562	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9563	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9564	}
9565	else
9566	{
9567	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9568	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9569	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9570	}
9571
9572	/* CS */
9573	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9574	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9575	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9576	/* SS */
9577	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9578	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9579	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9580	/* DS */
9581	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9582	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9583	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9584	/* ES */
9585	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9586	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9587	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9588	/* FS */
9589	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9590	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9591	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9592	/* GS */
9593	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9594	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9595	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9596	/* 64-bit capable CPUs. */
9597	#if HC_ARCH_BITS == 64
9598	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9599	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9600	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9601	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9602	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9603	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9604	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9605	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9606	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9607	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9608	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9609	#endif
9610	}
9611
9612	/*
9613	* TR.
9614	*/
9615	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9616	/* 64-bit capable CPUs. */
9617	#if HC_ARCH_BITS == 64
9618	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9619	#endif
9620	if (fLongModeGuest)
9621	{
9622	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9623	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9624	}
9625	else
9626	{
9627	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9628	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9629	VMX_IGS_TR_ATTR_TYPE_INVALID);
9630	}
9631	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9632	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9633	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9634	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9635	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9636	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9637	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9638	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9639
9640	/*
9641	* GDTR and IDTR.
9642	*/
9643	#if HC_ARCH_BITS == 64
9644	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9645	AssertRCBreak(rc);
9646	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9647
9648	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9649	AssertRCBreak(rc);
9650	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9651	#endif
9652
9653	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9654	AssertRCBreak(rc);
9655	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9656
9657	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9658	AssertRCBreak(rc);
9659	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9660
9661	/*
9662	* Guest Non-Register State.
9663	*/
9664	/* Activity State. */
9665	uint32_t u32ActivityState;
9666	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9667	AssertRCBreak(rc);
9668	HMVMX_CHECK_BREAK( !u32ActivityState
9669	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9670	VMX_IGS_ACTIVITY_STATE_INVALID);
9671	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9672	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9673	uint32_t u32IntrState;
9674	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9675	AssertRCBreak(rc);
9676	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9677	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9678	{
9679	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9680	}
9681
9682	/** @todo Activity state and injecting interrupts. Left as a todo since we
9683	* currently don't use activity states but ACTIVE. */
9684
9685	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9686	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9687
9688	/* Guest interruptibility-state. */
9689	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9690	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9691	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9692	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9693	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9694	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9695	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9696	if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
9697	{
9698	if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9699	{
9700	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9701	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9702	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9703	}
9704	else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9705	{
9706	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9707	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9708	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9709	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9710	}
9711	}
9712	/** @todo Assumes the processor is not in SMM. */
9713	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9714	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9715	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9716	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9717	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9718	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9719	&& VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9720	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9721	{
9722	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
9723	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9724	}
9725
9726	/* Pending debug exceptions. */
9727	#if HC_ARCH_BITS == 64
9728	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
9729	AssertRCBreak(rc);
9730	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9731	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9732	u32Val = u64Val; /* For pending debug exceptions checks below. */
9733	#else
9734	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
9735	AssertRCBreak(rc);
9736	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9737	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9738	#endif
9739
9740	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9741	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
9742	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9743	{
9744	if ( (u32Eflags & X86_EFL_TF)
9745	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9746	{
9747	/* Bit 14 is PendingDebug.BS. */
9748	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9749	}
9750	if ( !(u32Eflags & X86_EFL_TF)
9751	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9752	{
9753	/* Bit 14 is PendingDebug.BS. */
9754	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9755	}
9756	}
9757
9758	/* VMCS link pointer. */
9759	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9760	AssertRCBreak(rc);
9761	if (u64Val != UINT64_C(0xffffffffffffffff))
9762	{
9763	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9764	/** @todo Bits beyond the processor's physical-address width MBZ. */
9765	/** @todo 32-bit located in memory referenced by value of this field (as a
9766	* physical address) must contain the processor's VMCS revision ID. */
9767	/** @todo SMM checks. */
9768	}
9769
9770	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9771	* not using nested paging? */
9772	if ( pVM->hm.s.fNestedPaging
9773	&& !fLongModeGuest
9774	&& CPUMIsGuestInPAEModeEx(pCtx))
9775	{
9776	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9777	AssertRCBreak(rc);
9778	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9779
9780	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9781	AssertRCBreak(rc);
9782	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9783
9784	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9785	AssertRCBreak(rc);
9786	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9787
9788	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9789	AssertRCBreak(rc);
9790	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9791	}
9792
9793	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9794	if (uError == VMX_IGS_ERROR)
9795	uError = VMX_IGS_REASON_NOT_FOUND;
9796	} while (0);
9797
9798	pVCpu->hm.s.u32HMError = uError;
9799	return uError;
9800
9801	#undef HMVMX_ERROR_BREAK
9802	#undef HMVMX_CHECK_BREAK
9803	}
9804
9805
9806	/**
9807	* Setup the APIC-access page for virtualizing APIC access.
9808	*
9809	* This can cause a longjumps to R3 due to the acquisition of the PGM lock, hence
9810	* this not done as part of exporting guest state, see @bugref{8721}.
9811	*
9812	* @returns VBox status code.
9813	* @param pVCpu The cross context virtual CPU structure.
9814	*/
9815	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
9816	{
9817	PVM pVM = pVCpu->CTX_SUFF(pVM);
9818	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
9819
9820	Assert(PDMHasApic(pVM));
9821	Assert(u64MsrApicBase);
9822
9823	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
9824	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
9825
9826	/* Unalias any existing mapping. */
9827	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
9828	AssertRCReturn(rc, rc);
9829
9830	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
9831	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
9832	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
9833	AssertRCReturn(rc, rc);
9834
9835	/* Update the per-VCPU cache of the APIC base MSR. */
9836	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
9837	return VINF_SUCCESS;
9838	}
9839
9840
9841	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9842	/**
9843	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
9844	* nested-guest using hardware-assisted VMX.
9845	*
9846	* @param pVCpu The cross context virtual CPU structure.
9847	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
9848	* @param pVmcsInfoGst The guest VMCS info. object.
9849	*/
9850	static void hmR0VmxMergeMsrBitmapNested(PCVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
9851	{
9852	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
9853	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
9854	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
9855	Assert(pu64MsrBitmapNstGst);
9856	Assert(pu64MsrBitmapGst);
9857	Assert(pu64MsrBitmap);
9858
9859	/*
9860	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
9861	* MSR that is intercepted by the guest is also intercepted while executing the
9862	* nested-guest using hardware-assisted VMX.
9863	*/
9864	uint32_t const cbFrag = sizeof(uint64_t);
9865	uint32_t const cFrags = X86_PAGE_4K_SIZE / cbFrag;
9866	for (uint32_t i = 0; i <= cFrags; i++)
9867	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
9868	}
9869
9870
9871	/**
9872	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
9873	* hardware-assisted VMX execution of the nested-guest.
9874	*
9875	* For a guest, we don't modify these controls once we set up the VMCS.
9876	*
9877	* For nested-guests since the guest hypervisor provides these controls on every
9878	* nested-guest VM-entry and could potentially change them everytime we need to
9879	* merge them before every nested-guest VM-entry.
9880	*
9881	* @returns VBox status code.
9882	* @param pVCpu The cross context virtual CPU structure.
9883	*/
9884	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
9885	{
9886	PVM pVM = pVCpu->CTX_SUFF(pVM);
9887	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
9888	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
9889	Assert(pVmcsNstGst);
9890
9891	/*
9892	* Merge the controls with the requirements of the guest VMCS.
9893	*
9894	* We do not need to validate the nested-guest VMX features specified in the
9895	* nested-guest VMCS with the features supported by the physical CPU as it's
9896	* already done by the VMLAUNCH/VMRESUME instruction emulation.
9897	*
9898	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the
9899	* guest are derived from the VMX features supported by the physical CPU.
9900	*/
9901
9902	/* Pin-based VM-execution controls. */
9903	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
9904
9905	/* Processor-based VM-execution controls. */
9906	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
9907	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
9908	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
9909	\| VMX_PROC_CTLS_USE_TPR_SHADOW
9910	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
9911
9912	/* Secondary processor-based VM-execution controls. */
9913	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
9914	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
9915	\| VMX_PROC_CTLS2_INVPCID
9916	\| VMX_PROC_CTLS2_RDTSCP
9917	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
9918	\| VMX_PROC_CTLS2_APIC_REG_VIRT
9919	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
9920	\| VMX_PROC_CTLS2_VMFUNC));
9921
9922	/*
9923	* VM-entry controls:
9924	* These controls contains state that depends on the nested-guest state (primarily
9925	* EFER MSR) and is thus not constant through VMLAUNCH/VMRESUME and the nested-guest
9926	* VM-exit. Although the nested-hypervisor cannot change it, we need to in order to
9927	* properly continue executing the nested-guest if the EFER MSR changes but does not
9928	* cause a nested-guest VM-exits.
9929	*
9930	* VM-exit controls:
9931	* These controls specify the host state on return. We cannot use the controls from
9932	* the nested-hypervisor state as is as it would contain the guest state rather than
9933	* the host state. Since the host state is subject to change (e.g. preemption, trips
9934	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
9935	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
9936	*
9937	* VM-entry MSR-load:
9938	* The guest MSRs from the VM-entry MSR-load area are already loaded into the
9939	* guest-CPU context by the VMLAUNCH/VMRESUME instruction emulation.
9940	*
9941	* VM-exit MSR-store:
9942	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU
9943	* context back into the VM-exit MSR-store area.
9944	*
9945	* VM-exit MSR-load areas:
9946	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence,
9947	* we can entirely ignore what the nested-hypervisor wants to load here.
9948	*/
9949
9950	/*
9951	* Exception bitmap.
9952	*
9953	* We could remove #UD from the guest bitmap and merge it with the nested-guest
9954	* bitmap here (and avoid doing anything while exporting nested-guest state), but to
9955	* keep the code more flexible if intercepting exceptions become more dynamic in
9956	* the future we do it as part of exporting the nested-guest state.
9957	*/
9958	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
9959
9960	/*
9961	* CR0/CR4 guest/host mask.
9962	*
9963	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest
9964	* must cause VM-exits, so we need to merge them here.
9965	*/
9966	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
9967	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
9968
9969	/*
9970	* Page-fault error-code mask and match.
9971	*
9972	* Although we require unrestricted guest execution (and thereby nested-paging) for
9973	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
9974	* normally intercept #PFs, it might intercept them for debugging purposes.
9975	*
9976	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF
9977	* filters. If the outer guest is intercepting #PFs we must intercept all #PFs.
9978	*/
9979	uint32_t u32XcptPFMask;
9980	uint32_t u32XcptPFMatch;
9981	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
9982	{
9983	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
9984	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
9985	}
9986	else
9987	{
9988	u32XcptPFMask = 0;
9989	u32XcptPFMatch = 0;
9990	}
9991
9992	/*
9993	* Pause-Loop exiting.
9994	*/
9995	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
9996	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
9997
9998	/*
9999	* I/O Bitmap.
10000	*
10001	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we
10002	* always intercept all I/O port accesses.
10003	*/
10004	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10005
10006	/*
10007	* APIC-access page.
10008	*
10009	* The APIC-access page address has already been initialized while setting up the
10010	* nested-guest VMCS. In theory, even if the guest-physical address is invalid, it
10011	* should not be on any consequence to the host or to the guest for that matter, but
10012	* we only accept valid addresses verified by the VMLAUNCH/VMRESUME instruction
10013	* emulation to keep it simple.
10014	*/
10015
10016	/*
10017	* Virtual-APIC page and TPR threshold.
10018	*
10019	* We shall use the host-physical address of the virtual-APIC page in guest memory directly.
10020	* For this reason, we can access the virtual-APIC page of the nested-guest only using
10021	* PGM physical handlers as we must not assume a kernel virtual-address mapping exists and
10022	* requesting PGM for a mapping could be expensive/resource intensive (PGM mapping cache).
10023	*/
10024	RTHCPHYS HCPhysVirtApic = NIL_RTHCPHYS;
10025	uint32_t const u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10026	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10027	{
10028	int rc = PGMPhysGCPhys2HCPhys(pVM, pVmcsNstGst->u64AddrVirtApic.u, &HCPhysVirtApic);
10029
10030	/*
10031	* If the guest hypervisor has loaded crap into the virtual-APIC page field
10032	* we would fail to obtain a valid host-physical address for its guest-physical
10033	* address.
10034	*
10035	* We currently do not support this scenario. Maybe in the future if there is a
10036	* pressing need we can explore making this particular set of conditions work.
10037	* Right now we just cause a VM-entry failure.
10038	*
10039	* This has already been checked by VMLAUNCH/VMRESUME instruction emulation,
10040	* so should not really failure at the moment.
10041	*/
10042	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10043	}
10044	else
10045	{
10046	/*
10047	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10048	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10049	* be taken care of by EPT/shadow paging.
10050	*/
10051	if (pVM->hm.s.fAllow64BitGuests)
10052	{
10053	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10054	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10055	}
10056	}
10057
10058	/*
10059	* Validate basic assumptions.
10060	*/
10061	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10062	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10063	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10064	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10065
10066	/*
10067	* Commit it to the nested-guest VMCS.
10068	*/
10069	int rc = VINF_SUCCESS;
10070	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10071	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10072	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10073	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10074	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10075	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10076	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10077	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10078	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10079	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10080	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10081	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10082	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10083	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10084	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10085	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10086	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10087	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10088	{
10089	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10090	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10091	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10092	}
10093	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10094	{
10095	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10096	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10097	}
10098	AssertRCReturn(rc, rc);
10099
10100	/*
10101	* Update the nested-guest VMCS cache.
10102	*/
10103	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10104	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10105	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10106	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10107	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10108	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10109	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10110	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10111	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10112
10113	/*
10114	* MSR bitmap.
10115	*
10116	* The MSR bitmap address has already been initialized while setting up the
10117	* nested-guest VMCS, here we need to merge the MSR bitmaps.
10118	*/
10119	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10120	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10121
10122	return VINF_SUCCESS;
10123	}
10124	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10125
10126
10127	/**
10128	* Does the preparations before executing guest code in VT-x.
10129	*
10130	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10131	* recompiler/IEM. We must be cautious what we do here regarding committing
10132	* guest-state information into the VMCS assuming we assuredly execute the
10133	* guest in VT-x mode.
10134	*
10135	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10136	* the common-state (TRPM/forceflags), we must undo those changes so that the
10137	* recompiler/IEM can (and should) use them when it resumes guest execution.
10138	* Otherwise such operations must be done when we can no longer exit to ring-3.
10139	*
10140	* @returns Strict VBox status code (i.e. informational status codes too).
10141	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10142	* have been disabled.
10143	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10144	* double-fault into the guest.
10145	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10146	* dispatched directly.
10147	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10148	*
10149	* @param pVCpu The cross context virtual CPU structure.
10150	* @param pVmxTransient The VMX-transient structure.
10151	* @param fStepping Whether we are single-stepping the guest in the
10152	* hypervisor debugger. Makes us ignore some of the reasons
10153	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10154	* if event dispatching took place.
10155	*/
10156	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10157	{
10158	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10159
10160	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10161	if (pVmxTransient->fIsNestedGuest)
10162	{
10163	RT_NOREF2(pVCpu, fStepping);
10164	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10165	return VINF_EM_RESCHEDULE_REM;
10166	}
10167	#endif
10168
10169	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10170	PGMRZDynMapFlushAutoSet(pVCpu);
10171	#endif
10172
10173	/*
10174	* Check and process force flag actions, some of which might require us to go back to ring-3.
10175	*/
10176	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10177	if (rcStrict == VINF_SUCCESS)
10178	{ /* FFs don't get set all the time. */ }
10179	else
10180	return rcStrict;
10181
10182	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10183	/*
10184	* Switch to the nested-guest VMCS as we may have transitioned into executing
10185	* the nested-guest without leaving ring-0. Otherwise, if we came from ring-3
10186	* we would load the nested-guest VMCS while entering the VMX ring-0 session.
10187	*
10188	* We do this as late as possible to minimize (though not completely remove)
10189	* clearing/loading VMCS again due to premature trips to ring-3 above.
10190	*/
10191	if (pVmxTransient->fIsNestedGuest)
10192	{
10193	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10194	{
10195	/*
10196	* Ensure we have synced everything from the guest VMCS and also flag that
10197	* that we need to export the full (nested) guest-CPU context to the
10198	* nested-guest VMCS.
10199	*/
10200	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
10201	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST);
10202
10203	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
10204	int rc = hmR0VmxSwitchVmcs(&pVCpu->hm.s.vmx.VmcsInfo, &pVCpu->hm.s.vmx.VmcsInfoNstGst);
10205	if (RT_LIKELY(rc == VINF_SUCCESS))
10206	{
10207	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = true;
10208	ASMSetFlags(fEFlags);
10209	pVmxTransient->pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10210
10211	/*
10212	* We use a different VM-exit MSR-store area for the nested-guest. Hence,
10213	* flag that we need to update the host MSR values there. Even if we decide
10214	* in the future to share the VM-exit MSR-store area page with the guest,
10215	* if its content differs, we would have to update the host MSRs anyway.
10216	*/
10217	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
10218	Assert(!pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer); /** @todo NSTVMX: Paranoia remove later. */
10219	}
10220	else
10221	{
10222	ASMSetFlags(fEFlags);
10223	return rc;
10224	}
10225	}
10226
10227	/*
10228	* Merge guest VMCS controls with the nested-guest VMCS controls.
10229	*
10230	* Even if we have not executed the guest prior to this (e.g. when resuming
10231	* from a saved state), we should be okay with merging controls as we
10232	* initialize the guest VMCS controls as part of VM setup phase.
10233	*/
10234	if (!pVCpu->hm.s.vmx.fMergedNstGstCtls)
10235	{
10236	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10237	AssertRCReturn(rc, rc);
10238	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10239	}
10240	}
10241	#endif
10242
10243	/*
10244	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10245	* We look at the guest VMCS control here as we always set it when supported by
10246	* the physical CPU. Looking at the nested-guest control here would not be
10247	* possible because they are not merged yet.
10248	*/
10249	PVM pVM = pVCpu->CTX_SUFF(pVM);
10250	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10251	Assert(pVmcsInfo);
10252	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10253	&& (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10254	&& PDMHasApic(pVM))
10255	{
10256	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10257	AssertRCReturn(rc, rc);
10258	}
10259
10260	/*
10261	* Evaluate events to be injected into the guest.
10262	*
10263	* Events in TRPM can be injected without inspecting the guest state.
10264	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10265	* guest to cause a VM-exit the next time they are ready to receive the event.
10266	*/
10267	if (TRPMHasTrap(pVCpu))
10268	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10269
10270	uint32_t fIntrState;
10271	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10272
10273	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10274	/*
10275	* While evaluating pending events if something failed (unlikely) or if we were
10276	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10277	*/
10278	if ( rcStrict != VINF_SUCCESS
10279	\|\| ( pVmxTransient->fIsNestedGuest
10280	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)))
10281	return rcStrict;
10282	#endif
10283
10284	/*
10285	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10286	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10287	* also result in triple-faulting the VM.
10288	*
10289	* The above does not apply when executing a nested-guest (since unrestricted guest execution
10290	* is a requirement) regardless doing it avoid duplicating code elsewhere.
10291	*/
10292	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10293	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10294	{ /* likely */ }
10295	else
10296	{
10297	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10298	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10299	return rcStrict;
10300	}
10301
10302	/*
10303	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10304	* import CR3 themselves. We will need to update them here, as even as late as the above
10305	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10306	* the below force flags to be set.
10307	*/
10308	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10309	{
10310	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10311	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10312	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10313	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10314	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10315	}
10316	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10317	{
10318	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10319	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10320	}
10321
10322	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10323	/* Paranoia. */
10324	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10325	#endif
10326
10327	/*
10328	* No longjmps to ring-3 from this point on!!!
10329	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10330	* This also disables flushing of the R0-logger instance (if any).
10331	*/
10332	VMMRZCallRing3Disable(pVCpu);
10333
10334	/*
10335	* Export the guest state bits.
10336	*
10337	* We cannot perform longjmps while loading the guest state because we do not preserve the
10338	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10339	* CPU migration.
10340	*
10341	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10342	* registers. Hence, loading of the guest state needs to be done -after- injection of events.
10343	*/
10344	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10345	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10346	{ /* likely */ }
10347	else
10348	{
10349	VMMRZCallRing3Enable(pVCpu);
10350	return rcStrict;
10351	}
10352
10353	/*
10354	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10355	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10356	* preemption disabled for a while. Since this is purely to aid the
10357	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10358	* disable interrupt on NT.
10359	*
10360	* We need to check for force-flags that could've possible been altered since we last
10361	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10362	* see @bugref{6398}).
10363	*
10364	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10365	* to ring-3 before executing guest code.
10366	*/
10367	pVmxTransient->fEFlags = ASMIntDisableFlags();
10368
10369	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10370	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10371	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10372	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10373	{
10374	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10375	{
10376	pVCpu->hm.s.Event.fPending = false;
10377
10378	/*
10379	* We've injected any pending events. This is really the point of no return (to ring-3).
10380	*
10381	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10382	* returns from this function, so don't enable them here.
10383	*/
10384	return VINF_SUCCESS;
10385	}
10386
10387	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10388	rcStrict = VINF_EM_RAW_INTERRUPT;
10389	}
10390	else
10391	{
10392	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10393	rcStrict = VINF_EM_RAW_TO_R3;
10394	}
10395
10396	ASMSetFlags(pVmxTransient->fEFlags);
10397	VMMRZCallRing3Enable(pVCpu);
10398
10399	return rcStrict;
10400	}
10401
10402
10403	/**
10404	* Final preparations before executing guest code using hardware-assisted VMX.
10405	*
10406	* We can no longer get preempted to a different host CPU and there are no returns
10407	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10408	* failures), this function is not intended to fail sans unrecoverable hardware
10409	* errors.
10410	*
10411	* @param pVCpu The cross context virtual CPU structure.
10412	* @param pVmxTransient The VMX-transient structure.
10413	*
10414	* @remarks Called with preemption disabled.
10415	* @remarks No-long-jump zone!!!
10416	*/
10417	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10418	{
10419	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10420	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10421	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10422	Assert(!pVCpu->hm.s.Event.fPending);
10423
10424	/*
10425	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10426	*/
10427	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10428	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10429
10430	PVM pVM = pVCpu->CTX_SUFF(pVM);
10431	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10432
10433	if (!CPUMIsGuestFPUStateActive(pVCpu))
10434	{
10435	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10436	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10437	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10438	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10439	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10440	}
10441
10442	/*
10443	* Re-save the host state bits as we may've been preempted (only happens when
10444	* thread-context hooks are used or when the VM start function changes).
10445	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10446	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10447	* see @bugref{8432}.
10448	*
10449	* This may also happen when switching to/from a nested-guest VMCS without leaving
10450	* ring-0.
10451	*/
10452	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10453	{
10454	int rc = hmR0VmxExportHostState(pVCpu);
10455	AssertRC(rc);
10456	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
10457	}
10458	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10459
10460	/*
10461	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10462	*/
10463	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10464	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10465	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10466
10467	/*
10468	* Store status of the shared guest/host debug state at the time of VM-entry.
10469	*/
10470	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
10471	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
10472	{
10473	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
10474	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
10475	}
10476	else
10477	#endif
10478	{
10479	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10480	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10481	}
10482
10483	/*
10484	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10485	* more than one conditional check. The post-run side of our code shall determine
10486	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10487	*/
10488	if (pVmcsInfo->pbVirtApic)
10489	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10490
10491	/*
10492	* Update the host MSRs values in the VM-exit MSR-load area.
10493	*/
10494	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10495	{
10496	if (pVmcsInfo->cExitMsrLoad > 0)
10497	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10498	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10499	}
10500
10501	/*
10502	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10503	* VMX-preemption timer based on the next virtual sync clock deadline.
10504	*/
10505	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10506	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10507	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10508	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10509	{
10510	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10511	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10512	}
10513
10514	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10515	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10516	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10517	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
10518
10519	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10520
10521	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10522	as we're about to start executing the guest . */
10523
10524	/*
10525	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10526	*
10527	* This is done this late as updating the TSC offsetting/preemption timer above
10528	* figures out if we can skip intercepting RDTSCP by calculating the number of
10529	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10530	*/
10531	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10532	{
10533	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10534	{
10535	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10536	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10537	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10538	AssertRC(rc);
10539	}
10540	else
10541	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
10542	}
10543
10544	#ifdef VBOX_STRICT
10545	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo);
10546	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10547	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo));
10548	#endif
10549
10550	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10551	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10552	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs. */
10553	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10554	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10555	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10556	#endif
10557	}
10558
10559
10560	/**
10561	* First C routine invoked after running guest code using hardware-assisted VMX.
10562	*
10563	* @param pVCpu The cross context virtual CPU structure.
10564	* @param pVmxTransient The VMX-transient structure.
10565	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10566	*
10567	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10568	*
10569	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10570	* unconditionally when it is safe to do so.
10571	*/
10572	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10573	{
10574	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10575
10576	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10577	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10578	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10579	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10580	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10581	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10582
10583	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10584	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10585	{
10586	uint64_t uGstTsc;
10587	if (!pVmxTransient->fIsNestedGuest)
10588	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10589	else
10590	{
10591	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10592	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10593	}
10594	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10595	}
10596
10597	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10598	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
10599	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10600
10601	#if HC_ARCH_BITS == 64
10602	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10603	#endif
10604	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
10605	/* The 64-on-32 switcher maintains VMCS-launch state on its own
10606	and we need to leave it alone here. */
10607	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
10608	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10609	#else
10610	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10611	#endif
10612	#ifdef VBOX_STRICT
10613	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10614	#endif
10615	Assert(!ASMIntAreEnabled());
10616	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10617	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10618
10619	/*
10620	* Save the basic VM-exit reason and check if the VM-entry failed.
10621	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10622	*/
10623	uint32_t uExitReason;
10624	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
10625	AssertRC(rc);
10626	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
10627	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
10628
10629	/*
10630	* Check if VMLAUNCH/VMRESUME succeeded.
10631	* If this failed, we cause a guru meditation and cease further execution.
10632	*/
10633	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
10634	{
10635	/*
10636	* Update the VM-exit history array here even if the VM-entry failed due to:
10637	* - Invalid guest state.
10638	* - MSR loading.
10639	* - Machine-check event.
10640	*
10641	* In any of the above cases we will still have a "valid" VM-exit reason
10642	* despite @a fVMEntryFailed being false.
10643	*
10644	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
10645	*
10646	* Note! We don't have CS or RIP at this point. Will probably address that later
10647	* by amending the history entry added here.
10648	*/
10649	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
10650	UINT64_MAX, uHostTsc);
10651
10652	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
10653	{
10654	VMMRZCallRing3Enable(pVCpu);
10655
10656	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10657	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10658
10659	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
10660	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10661	AssertRC(rc);
10662	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
10663	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
10664	AssertRC(rc);
10665	#else
10666	/*
10667	* Import the guest-interruptibility state always as we need it while evaluating
10668	* injecting events on re-entry.
10669	*
10670	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
10671	* checking for real-mode while exporting the state because all bits that cause
10672	* mode changes wrt CR0 are intercepted.
10673	*/
10674	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
10675	AssertRC(rc);
10676	#endif
10677
10678	/*
10679	* Sync the TPR shadow with our APIC state.
10680	*/
10681	if ( !pVmxTransient->fIsNestedGuest
10682	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
10683	{
10684	Assert(pVmcsInfo->pbVirtApic);
10685	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
10686	{
10687	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
10688	AssertRC(rc);
10689	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
10690	}
10691	}
10692
10693	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10694	return;
10695	}
10696	}
10697	else
10698	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
10699
10700	VMMRZCallRing3Enable(pVCpu);
10701	}
10702
10703
10704	/**
10705	* Runs the guest code using hardware-assisted VMX the normal way.
10706	*
10707	* @returns VBox status code.
10708	* @param pVCpu The cross context virtual CPU structure.
10709	* @param pcLoops Pointer to the number of executed loops.
10710	*/
10711	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
10712	{
10713	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10714	Assert(pcLoops);
10715	Assert(*pcLoops <= cMaxResumeLoops);
10716
10717	VMXTRANSIENT VmxTransient;
10718	RT_ZERO(VmxTransient);
10719	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10720
10721	/* Paranoia. */
10722	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
10723	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10724
10725	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10726	for (;;)
10727	{
10728	Assert(!HMR0SuspendPending());
10729	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10730	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10731
10732	/*
10733	* Preparatory work for running nested-guest code, this may force us to
10734	* return to ring-3.
10735	*
10736	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10737	*/
10738	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10739	if (rcStrict != VINF_SUCCESS)
10740	break;
10741
10742	/* Interrupts are disabled at this point! */
10743	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10744	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10745	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10746	/* Interrupts are re-enabled at this point! */
10747
10748	/*
10749	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10750	*/
10751	if (RT_SUCCESS(rcRun))
10752	{ /* very likely */ }
10753	else
10754	{
10755	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10756	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10757	return rcRun;
10758	}
10759
10760	/*
10761	* Profile the VM-exit.
10762	*/
10763	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10764	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10765	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10766	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10767	HMVMX_START_EXIT_DISPATCH_PROF();
10768
10769	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10770
10771	/*
10772	* Handle the VM-exit.
10773	*/
10774	#ifdef HMVMX_USE_FUNCTION_TABLE
10775	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
10776	#else
10777	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
10778	#endif
10779	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10780	if (rcStrict == VINF_SUCCESS)
10781	{
10782	if (++(*pcLoops) <= cMaxResumeLoops)
10783	continue;
10784	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10785	rcStrict = VINF_EM_RAW_INTERRUPT;
10786	}
10787	break;
10788	}
10789
10790	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10791	return rcStrict;
10792	}
10793
10794	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10795	/**
10796	* Runs the nested-guest code using hardware-assisted VMX.
10797	*
10798	* @returns VBox status code.
10799	* @param pVCpu The cross context virtual CPU structure.
10800	* @param pcLoops Pointer to the number of executed loops.
10801	*
10802	* @sa hmR0VmxRunGuestCodeNormal().
10803	*/
10804	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
10805	{
10806	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10807	Assert(pcLoops);
10808	Assert(*pcLoops <= cMaxResumeLoops);
10809
10810	VMXTRANSIENT VmxTransient;
10811	RT_ZERO(VmxTransient);
10812	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10813	VmxTransient.fIsNestedGuest = true;
10814
10815	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10816	for (;;)
10817	{
10818	Assert(!HMR0SuspendPending());
10819	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10820	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10821
10822	/*
10823	* Preparatory work for running guest code, this may force us to
10824	* return to ring-3.
10825	*
10826	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10827	*/
10828	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10829	if (rcStrict != VINF_SUCCESS)
10830	break;
10831
10832	/* Interrupts are disabled at this point! */
10833	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10834	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10835	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10836	/* Interrupts are re-enabled at this point! */
10837
10838	/*
10839	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10840	*/
10841	if (RT_SUCCESS(rcRun))
10842	{ /* very likely */ }
10843	else
10844	{
10845	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10846	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10847	return rcRun;
10848	}
10849
10850	/*
10851	* Profile the VM-exit.
10852	*/
10853	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10854	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10855	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10856	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10857	HMVMX_START_EXIT_DISPATCH_PROF();
10858
10859	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10860
10861	/*
10862	* Handle the VM-exit.
10863	*/
10864	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
10865	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10866	if ( rcStrict == VINF_SUCCESS
10867	&& CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10868	{
10869	if (++(*pcLoops) <= cMaxResumeLoops)
10870	continue;
10871	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10872	rcStrict = VINF_EM_RAW_INTERRUPT;
10873	}
10874	break;
10875	}
10876
10877	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10878	return rcStrict;
10879	}
10880	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10881
10882
10883	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
10884	* probes.
10885	*
10886	* The following few functions and associated structure contains the bloat
10887	* necessary for providing detailed debug events and dtrace probes as well as
10888	* reliable host side single stepping. This works on the principle of
10889	* "subclassing" the normal execution loop and workers. We replace the loop
10890	* method completely and override selected helpers to add necessary adjustments
10891	* to their core operation.
10892	*
10893	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
10894	* any performance for debug and analysis features.
10895	*
10896	* @{
10897	*/
10898
10899	/**
10900	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
10901	* the debug run loop.
10902	*/
10903	typedef struct VMXRUNDBGSTATE
10904	{
10905	/** The RIP we started executing at. This is for detecting that we stepped. */
10906	uint64_t uRipStart;
10907	/** The CS we started executing with. */
10908	uint16_t uCsStart;
10909
10910	/** Whether we've actually modified the 1st execution control field. */
10911	bool fModifiedProcCtls : 1;
10912	/** Whether we've actually modified the 2nd execution control field. */
10913	bool fModifiedProcCtls2 : 1;
10914	/** Whether we've actually modified the exception bitmap. */
10915	bool fModifiedXcptBitmap : 1;
10916
10917	/** We desire the modified the CR0 mask to be cleared. */
10918	bool fClearCr0Mask : 1;
10919	/** We desire the modified the CR4 mask to be cleared. */
10920	bool fClearCr4Mask : 1;
10921	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
10922	uint32_t fCpe1Extra;
10923	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
10924	uint32_t fCpe1Unwanted;
10925	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
10926	uint32_t fCpe2Extra;
10927	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
10928	uint32_t bmXcptExtra;
10929	/** The sequence number of the Dtrace provider settings the state was
10930	* configured against. */
10931	uint32_t uDtraceSettingsSeqNo;
10932	/** VM-exits to check (one bit per VM-exit). */
10933	uint32_t bmExitsToCheck[3];
10934
10935	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
10936	uint32_t fProcCtlsInitial;
10937	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
10938	uint32_t fProcCtls2Initial;
10939	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
10940	uint32_t bmXcptInitial;
10941	} VMXRUNDBGSTATE;
10942	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
10943	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
10944
10945
10946	/**
10947	* Initializes the VMXRUNDBGSTATE structure.
10948	*
10949	* @param pVCpu The cross context virtual CPU structure of the
10950	* calling EMT.
10951	* @param pVmxTransient The VMX-transient structure.
10952	* @param pDbgState The debug state to initialize.
10953	*/
10954	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
10955	{
10956	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
10957	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
10958
10959	pDbgState->fModifiedProcCtls = false;
10960	pDbgState->fModifiedProcCtls2 = false;
10961	pDbgState->fModifiedXcptBitmap = false;
10962	pDbgState->fClearCr0Mask = false;
10963	pDbgState->fClearCr4Mask = false;
10964	pDbgState->fCpe1Extra = 0;
10965	pDbgState->fCpe1Unwanted = 0;
10966	pDbgState->fCpe2Extra = 0;
10967	pDbgState->bmXcptExtra = 0;
10968	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
10969	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
10970	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
10971	}
10972
10973
10974	/**
10975	* Updates the VMSC fields with changes requested by @a pDbgState.
10976	*
10977	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
10978	* immediately before executing guest code, i.e. when interrupts are disabled.
10979	* We don't check status codes here as we cannot easily assert or return in the
10980	* latter case.
10981	*
10982	* @param pVCpu The cross context virtual CPU structure.
10983	* @param pVmxTransient The VMX-transient structure.
10984	* @param pDbgState The debug state.
10985	*/
10986	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
10987	{
10988	/*
10989	* Ensure desired flags in VMCS control fields are set.
10990	* (Ignoring write failure here, as we're committed and it's just debug extras.)
10991	*
10992	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
10993	* there should be no stale data in pCtx at this point.
10994	*/
10995	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10996	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
10997	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
10998	{
10999	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11000	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11001	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11002	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11003	pDbgState->fModifiedProcCtls = true;
11004	}
11005
11006	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11007	{
11008	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11009	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11010	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11011	pDbgState->fModifiedProcCtls2 = true;
11012	}
11013
11014	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11015	{
11016	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11017	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11018	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11019	pDbgState->fModifiedXcptBitmap = true;
11020	}
11021
11022	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11023	{
11024	pVmcsInfo->u64Cr0Mask = 0;
11025	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
11026	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11027	}
11028
11029	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11030	{
11031	pVmcsInfo->u64Cr4Mask = 0;
11032	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
11033	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11034	}
11035
11036	NOREF(pVCpu);
11037	}
11038
11039
11040	/**
11041	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11042	* re-entry next time around.
11043	*
11044	* @returns Strict VBox status code (i.e. informational status codes too).
11045	* @param pVCpu The cross context virtual CPU structure.
11046	* @param pVmxTransient The VMX-transient structure.
11047	* @param pDbgState The debug state.
11048	* @param rcStrict The return code from executing the guest using single
11049	* stepping.
11050	*/
11051	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11052	VBOXSTRICTRC rcStrict)
11053	{
11054	/*
11055	* Restore VM-exit control settings as we may not reenter this function the
11056	* next time around.
11057	*/
11058	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11059
11060	/* We reload the initial value, trigger what we can of recalculations the
11061	next time around. From the looks of things, that's all that's required atm. */
11062	if (pDbgState->fModifiedProcCtls)
11063	{
11064	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11065	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11066	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11067	AssertRCReturn(rc2, rc2);
11068	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11069	}
11070
11071	/* We're currently the only ones messing with this one, so just restore the
11072	cached value and reload the field. */
11073	if ( pDbgState->fModifiedProcCtls2
11074	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11075	{
11076	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11077	AssertRCReturn(rc2, rc2);
11078	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11079	}
11080
11081	/* If we've modified the exception bitmap, we restore it and trigger
11082	reloading and partial recalculation the next time around. */
11083	if (pDbgState->fModifiedXcptBitmap)
11084	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11085
11086	return rcStrict;
11087	}
11088
11089
11090	/**
11091	* Configures VM-exit controls for current DBGF and DTrace settings.
11092	*
11093	* This updates @a pDbgState and the VMCS execution control fields to reflect
11094	* the necessary VM-exits demanded by DBGF and DTrace.
11095	*
11096	* @param pVCpu The cross context virtual CPU structure.
11097	* @param pVmxTransient The VMX-transient structure. May update
11098	* fUpdatedTscOffsettingAndPreemptTimer.
11099	* @param pDbgState The debug state.
11100	*/
11101	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11102	{
11103	/*
11104	* Take down the dtrace serial number so we can spot changes.
11105	*/
11106	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11107	ASMCompilerBarrier();
11108
11109	/*
11110	* We'll rebuild most of the middle block of data members (holding the
11111	* current settings) as we go along here, so start by clearing it all.
11112	*/
11113	pDbgState->bmXcptExtra = 0;
11114	pDbgState->fCpe1Extra = 0;
11115	pDbgState->fCpe1Unwanted = 0;
11116	pDbgState->fCpe2Extra = 0;
11117	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11118	pDbgState->bmExitsToCheck[i] = 0;
11119
11120	/*
11121	* Software interrupts (INT XXh) - no idea how to trigger these...
11122	*/
11123	PVM pVM = pVCpu->CTX_SUFF(pVM);
11124	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11125	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11126	{
11127	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11128	}
11129
11130	/*
11131	* INT3 breakpoints - triggered by #BP exceptions.
11132	*/
11133	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11134	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11135
11136	/*
11137	* Exception bitmap and XCPT events+probes.
11138	*/
11139	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11140	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11141	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11142
11143	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11144	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11145	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11146	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11147	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11148	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11149	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11150	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11151	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11152	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11153	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11154	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11155	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11156	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11157	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11158	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11159	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11160	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11161
11162	if (pDbgState->bmXcptExtra)
11163	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11164
11165	/*
11166	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11167	*
11168	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11169	* So, when adding/changing/removing please don't forget to update it.
11170	*
11171	* Some of the macros are picking up local variables to save horizontal space,
11172	* (being able to see it in a table is the lesser evil here).
11173	*/
11174	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11175	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11176	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11177	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11178	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11179	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11180	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11181	} else do { } while (0)
11182	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11183	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11184	{ \
11185	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11186	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11187	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11188	} else do { } while (0)
11189	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11190	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11191	{ \
11192	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11193	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11194	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11195	} else do { } while (0)
11196	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11197	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11198	{ \
11199	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11200	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11201	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11202	} else do { } while (0)
11203
11204	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11205	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11206	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11207	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11208	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11209
11210	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11211	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11212	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11213	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11214	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11215	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11216	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11217	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11218	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11219	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11220	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11221	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11222	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11223	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11224	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11225	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11226	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11227	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11228	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11229	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11230	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11231	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11232	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11233	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11234	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11235	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11236	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11237	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11238	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11239	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11240	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11241	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11242	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11243	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11244	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11245	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11246
11247	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11248	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11249	{
11250	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11251	\| CPUMCTX_EXTRN_APIC_TPR);
11252	AssertRC(rc);
11253
11254	#if 0 /** @todo fix me */
11255	pDbgState->fClearCr0Mask = true;
11256	pDbgState->fClearCr4Mask = true;
11257	#endif
11258	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11259	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11260	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11261	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11262	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11263	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11264	require clearing here and in the loop if we start using it. */
11265	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11266	}
11267	else
11268	{
11269	if (pDbgState->fClearCr0Mask)
11270	{
11271	pDbgState->fClearCr0Mask = false;
11272	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11273	}
11274	if (pDbgState->fClearCr4Mask)
11275	{
11276	pDbgState->fClearCr4Mask = false;
11277	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11278	}
11279	}
11280	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11281	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11282
11283	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11284	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11285	{
11286	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11287	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11288	}
11289	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11290	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11291
11292	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11293	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11294	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11295	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11296	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11297	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11298	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11299	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11300	#if 0 /** @todo too slow, fix handler. */
11301	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11302	#endif
11303	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11304
11305	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11306	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11307	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11308	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11309	{
11310	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11311	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11312	}
11313	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11314	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11315	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11316	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11317
11318	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11319	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11320	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11321	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11322	{
11323	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11324	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11325	}
11326	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11327	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11328	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11329	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11330
11331	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11332	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11333	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11334	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11335	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11336	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11337	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11338	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11339	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11340	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11341	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11342	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11343	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11344	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11345	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11346	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11347	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11348	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11349	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11350	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11351	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11352	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11353
11354	#undef IS_EITHER_ENABLED
11355	#undef SET_ONLY_XBM_IF_EITHER_EN
11356	#undef SET_CPE1_XBM_IF_EITHER_EN
11357	#undef SET_CPEU_XBM_IF_EITHER_EN
11358	#undef SET_CPE2_XBM_IF_EITHER_EN
11359
11360	/*
11361	* Sanitize the control stuff.
11362	*/
11363	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11364	if (pDbgState->fCpe2Extra)
11365	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11366	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11367	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11368	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11369	{
11370	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11371	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11372	}
11373
11374	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11375	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11376	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11377	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11378	}
11379
11380
11381	/**
11382	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11383	* appropriate.
11384	*
11385	* The caller has checked the VM-exit against the
11386	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11387	* already, so we don't have to do that either.
11388	*
11389	* @returns Strict VBox status code (i.e. informational status codes too).
11390	* @param pVCpu The cross context virtual CPU structure.
11391	* @param pVmxTransient The VMX-transient structure.
11392	* @param uExitReason The VM-exit reason.
11393	*
11394	* @remarks The name of this function is displayed by dtrace, so keep it short
11395	* and to the point. No longer than 33 chars long, please.
11396	*/
11397	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11398	{
11399	/*
11400	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11401	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11402	*
11403	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11404	* does. Must add/change/remove both places. Same ordering, please.
11405	*
11406	* Added/removed events must also be reflected in the next section
11407	* where we dispatch dtrace events.
11408	*/
11409	bool fDtrace1 = false;
11410	bool fDtrace2 = false;
11411	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11412	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11413	uint32_t uEventArg = 0;
11414	#define SET_EXIT(a_EventSubName) \
11415	do { \
11416	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11417	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11418	} while (0)
11419	#define SET_BOTH(a_EventSubName) \
11420	do { \
11421	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11422	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11423	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11424	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11425	} while (0)
11426	switch (uExitReason)
11427	{
11428	case VMX_EXIT_MTF:
11429	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11430
11431	case VMX_EXIT_XCPT_OR_NMI:
11432	{
11433	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11434	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11435	{
11436	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11437	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11438	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11439	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11440	{
11441	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11442	{
11443	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11444	uEventArg = pVmxTransient->uExitIntErrorCode;
11445	}
11446	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11447	switch (enmEvent1)
11448	{
11449	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11450	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11451	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11452	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11453	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11454	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11455	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11456	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11457	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11458	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11459	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11460	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11461	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11462	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11463	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11464	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11465	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11466	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11467	default: break;
11468	}
11469	}
11470	else
11471	AssertFailed();
11472	break;
11473
11474	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11475	uEventArg = idxVector;
11476	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11477	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11478	break;
11479	}
11480	break;
11481	}
11482
11483	case VMX_EXIT_TRIPLE_FAULT:
11484	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11485	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11486	break;
11487	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11488	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11489	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11490	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11491	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11492
11493	/* Instruction specific VM-exits: */
11494	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11495	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11496	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11497	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11498	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11499	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11500	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11501	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11502	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11503	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11504	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11505	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11506	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11507	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11508	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11509	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11510	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11511	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11512	case VMX_EXIT_MOV_CRX:
11513	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11514	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11515	SET_BOTH(CRX_READ);
11516	else
11517	SET_BOTH(CRX_WRITE);
11518	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11519	break;
11520	case VMX_EXIT_MOV_DRX:
11521	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11522	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11523	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11524	SET_BOTH(DRX_READ);
11525	else
11526	SET_BOTH(DRX_WRITE);
11527	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11528	break;
11529	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11530	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11531	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11532	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11533	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11534	case VMX_EXIT_GDTR_IDTR_ACCESS:
11535	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11536	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11537	{
11538	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11539	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11540	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11541	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11542	}
11543	break;
11544
11545	case VMX_EXIT_LDTR_TR_ACCESS:
11546	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11547	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11548	{
11549	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11550	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11551	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11552	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11553	}
11554	break;
11555
11556	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11557	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11558	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11559	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11560	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11561	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11562	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11563	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11564	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11565	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11566	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11567
11568	/* Events that aren't relevant at this point. */
11569	case VMX_EXIT_EXT_INT:
11570	case VMX_EXIT_INT_WINDOW:
11571	case VMX_EXIT_NMI_WINDOW:
11572	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11573	case VMX_EXIT_PREEMPT_TIMER:
11574	case VMX_EXIT_IO_INSTR:
11575	break;
11576
11577	/* Errors and unexpected events. */
11578	case VMX_EXIT_INIT_SIGNAL:
11579	case VMX_EXIT_SIPI:
11580	case VMX_EXIT_IO_SMI:
11581	case VMX_EXIT_SMI:
11582	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11583	case VMX_EXIT_ERR_MSR_LOAD:
11584	case VMX_EXIT_ERR_MACHINE_CHECK:
11585	break;
11586
11587	default:
11588	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11589	break;
11590	}
11591	#undef SET_BOTH
11592	#undef SET_EXIT
11593
11594	/*
11595	* Dtrace tracepoints go first. We do them here at once so we don't
11596	* have to copy the guest state saving and stuff a few dozen times.
11597	* Down side is that we've got to repeat the switch, though this time
11598	* we use enmEvent since the probes are a subset of what DBGF does.
11599	*/
11600	if (fDtrace1 \|\| fDtrace2)
11601	{
11602	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11603	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11604	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11605	switch (enmEvent1)
11606	{
11607	/** @todo consider which extra parameters would be helpful for each probe. */
11608	case DBGFEVENT_END: break;
11609	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11610	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11611	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11612	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11613	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11614	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11615	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11616	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11617	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
11618	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
11619	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
11620	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
11621	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
11622	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
11623	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
11624	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
11625	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
11626	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
11627	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11628	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11629	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
11630	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
11631	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
11632	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
11633	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
11634	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
11635	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
11636	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11637	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11638	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11639	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11640	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11641	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
11642	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11643	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
11644	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
11645	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
11646	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
11647	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
11648	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
11649	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
11650	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
11651	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
11652	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
11653	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
11654	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
11655	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
11656	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
11657	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
11658	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
11659	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
11660	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
11661	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
11662	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
11663	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
11664	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
11665	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
11666	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
11667	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
11668	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
11669	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
11670	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
11671	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
11672	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
11673	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
11674	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
11675	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
11676	}
11677	switch (enmEvent2)
11678	{
11679	/** @todo consider which extra parameters would be helpful for each probe. */
11680	case DBGFEVENT_END: break;
11681	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
11682	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11683	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
11684	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
11685	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
11686	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
11687	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
11688	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
11689	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
11690	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11691	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11692	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11693	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11694	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11695	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
11696	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11697	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
11698	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
11699	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
11700	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
11701	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
11702	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
11703	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
11704	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
11705	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
11706	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
11707	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
11708	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
11709	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
11710	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
11711	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
11712	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
11713	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
11714	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
11715	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
11716	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
11717	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
11718	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
11719	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
11720	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
11721	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
11722	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
11723	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
11724	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
11725	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
11726	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
11727	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
11728	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
11729	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
11730	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
11731	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
11732	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
11733	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
11734	}
11735	}
11736
11737	/*
11738	* Fire of the DBGF event, if enabled (our check here is just a quick one,
11739	* the DBGF call will do a full check).
11740	*
11741	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
11742	* Note! If we have to events, we prioritize the first, i.e. the instruction
11743	* one, in order to avoid event nesting.
11744	*/
11745	PVM pVM = pVCpu->CTX_SUFF(pVM);
11746	if ( enmEvent1 != DBGFEVENT_END
11747	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
11748	{
11749	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11750	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
11751	if (rcStrict != VINF_SUCCESS)
11752	return rcStrict;
11753	}
11754	else if ( enmEvent2 != DBGFEVENT_END
11755	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
11756	{
11757	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11758	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
11759	if (rcStrict != VINF_SUCCESS)
11760	return rcStrict;
11761	}
11762
11763	return VINF_SUCCESS;
11764	}
11765
11766
11767	/**
11768	* Single-stepping VM-exit filtering.
11769	*
11770	* This is preprocessing the VM-exits and deciding whether we've gotten far
11771	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
11772	* handling is performed.
11773	*
11774	* @returns Strict VBox status code (i.e. informational status codes too).
11775	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
11776	* @param pVmxTransient The VMX-transient structure.
11777	* @param pDbgState The debug state.
11778	*/
11779	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11780	{
11781	/*
11782	* Expensive (saves context) generic dtrace VM-exit probe.
11783	*/
11784	uint32_t const uExitReason = pVmxTransient->uExitReason;
11785	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
11786	{ /* more likely */ }
11787	else
11788	{
11789	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11790	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11791	AssertRC(rc);
11792	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
11793	}
11794
11795	/*
11796	* Check for host NMI, just to get that out of the way.
11797	*/
11798	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
11799	{ /* normally likely */ }
11800	else
11801	{
11802	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11803	AssertRCReturn(rc2, rc2);
11804	uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11805	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11806	return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
11807	}
11808
11809	/*
11810	* Check for single stepping event if we're stepping.
11811	*/
11812	if (pVCpu->hm.s.fSingleInstruction)
11813	{
11814	switch (uExitReason)
11815	{
11816	case VMX_EXIT_MTF:
11817	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11818
11819	/* Various events: */
11820	case VMX_EXIT_XCPT_OR_NMI:
11821	case VMX_EXIT_EXT_INT:
11822	case VMX_EXIT_TRIPLE_FAULT:
11823	case VMX_EXIT_INT_WINDOW:
11824	case VMX_EXIT_NMI_WINDOW:
11825	case VMX_EXIT_TASK_SWITCH:
11826	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11827	case VMX_EXIT_APIC_ACCESS:
11828	case VMX_EXIT_EPT_VIOLATION:
11829	case VMX_EXIT_EPT_MISCONFIG:
11830	case VMX_EXIT_PREEMPT_TIMER:
11831
11832	/* Instruction specific VM-exits: */
11833	case VMX_EXIT_CPUID:
11834	case VMX_EXIT_GETSEC:
11835	case VMX_EXIT_HLT:
11836	case VMX_EXIT_INVD:
11837	case VMX_EXIT_INVLPG:
11838	case VMX_EXIT_RDPMC:
11839	case VMX_EXIT_RDTSC:
11840	case VMX_EXIT_RSM:
11841	case VMX_EXIT_VMCALL:
11842	case VMX_EXIT_VMCLEAR:
11843	case VMX_EXIT_VMLAUNCH:
11844	case VMX_EXIT_VMPTRLD:
11845	case VMX_EXIT_VMPTRST:
11846	case VMX_EXIT_VMREAD:
11847	case VMX_EXIT_VMRESUME:
11848	case VMX_EXIT_VMWRITE:
11849	case VMX_EXIT_VMXOFF:
11850	case VMX_EXIT_VMXON:
11851	case VMX_EXIT_MOV_CRX:
11852	case VMX_EXIT_MOV_DRX:
11853	case VMX_EXIT_IO_INSTR:
11854	case VMX_EXIT_RDMSR:
11855	case VMX_EXIT_WRMSR:
11856	case VMX_EXIT_MWAIT:
11857	case VMX_EXIT_MONITOR:
11858	case VMX_EXIT_PAUSE:
11859	case VMX_EXIT_GDTR_IDTR_ACCESS:
11860	case VMX_EXIT_LDTR_TR_ACCESS:
11861	case VMX_EXIT_INVEPT:
11862	case VMX_EXIT_RDTSCP:
11863	case VMX_EXIT_INVVPID:
11864	case VMX_EXIT_WBINVD:
11865	case VMX_EXIT_XSETBV:
11866	case VMX_EXIT_RDRAND:
11867	case VMX_EXIT_INVPCID:
11868	case VMX_EXIT_VMFUNC:
11869	case VMX_EXIT_RDSEED:
11870	case VMX_EXIT_XSAVES:
11871	case VMX_EXIT_XRSTORS:
11872	{
11873	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11874	AssertRCReturn(rc, rc);
11875	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
11876	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
11877	return VINF_EM_DBG_STEPPED;
11878	break;
11879	}
11880
11881	/* Errors and unexpected events: */
11882	case VMX_EXIT_INIT_SIGNAL:
11883	case VMX_EXIT_SIPI:
11884	case VMX_EXIT_IO_SMI:
11885	case VMX_EXIT_SMI:
11886	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11887	case VMX_EXIT_ERR_MSR_LOAD:
11888	case VMX_EXIT_ERR_MACHINE_CHECK:
11889	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
11890	break;
11891
11892	default:
11893	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11894	break;
11895	}
11896	}
11897
11898	/*
11899	* Check for debugger event breakpoints and dtrace probes.
11900	*/
11901	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
11902	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
11903	{
11904	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
11905	if (rcStrict != VINF_SUCCESS)
11906	return rcStrict;
11907	}
11908
11909	/*
11910	* Normal processing.
11911	*/
11912	#ifdef HMVMX_USE_FUNCTION_TABLE
11913	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
11914	#else
11915	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
11916	#endif
11917	}
11918
11919
11920	/**
11921	* Single steps guest code using hardware-assisted VMX.
11922	*
11923	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
11924	* but single-stepping through the hypervisor debugger.
11925	*
11926	* @returns Strict VBox status code (i.e. informational status codes too).
11927	* @param pVCpu The cross context virtual CPU structure.
11928	* @param pcLoops Pointer to the number of executed loops.
11929	*
11930	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
11931	*/
11932	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
11933	{
11934	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11935	Assert(pcLoops);
11936	Assert(*pcLoops <= cMaxResumeLoops);
11937
11938	VMXTRANSIENT VmxTransient;
11939	RT_ZERO(VmxTransient);
11940	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11941
11942	/* Set HMCPU indicators. */
11943	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
11944	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
11945	pVCpu->hm.s.fDebugWantRdTscExit = false;
11946	pVCpu->hm.s.fUsingDebugLoop = true;
11947
11948	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
11949	VMXRUNDBGSTATE DbgState;
11950	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
11951	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
11952
11953	/*
11954	* The loop.
11955	*/
11956	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11957	for (;;)
11958	{
11959	Assert(!HMR0SuspendPending());
11960	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11961	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11962	bool fStepping = pVCpu->hm.s.fSingleInstruction;
11963
11964	/* Set up VM-execution controls the next two can respond to. */
11965	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
11966
11967	/*
11968	* Preparatory work for running guest code, this may force us to
11969	* return to ring-3.
11970	*
11971	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11972	*/
11973	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
11974	if (rcStrict != VINF_SUCCESS)
11975	break;
11976
11977	/* Interrupts are disabled at this point! */
11978	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11979
11980	/* Override any obnoxious code in the above two calls. */
11981	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
11982
11983	/*
11984	* Finally execute the guest.
11985	*/
11986	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11987
11988	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11989	/* Interrupts are re-enabled at this point! */
11990
11991	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
11992	if (RT_SUCCESS(rcRun))
11993	{ /* very likely */ }
11994	else
11995	{
11996	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11997	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11998	return rcRun;
11999	}
12000
12001	/* Profile the VM-exit. */
12002	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12003	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12004	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12005	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12006	HMVMX_START_EXIT_DISPATCH_PROF();
12007
12008	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12009
12010	/*
12011	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12012	*/
12013	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12014	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12015	if (rcStrict != VINF_SUCCESS)
12016	break;
12017	if (++(*pcLoops) > cMaxResumeLoops)
12018	{
12019	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12020	rcStrict = VINF_EM_RAW_INTERRUPT;
12021	break;
12022	}
12023
12024	/*
12025	* Stepping: Did the RIP change, if so, consider it a single step.
12026	* Otherwise, make sure one of the TFs gets set.
12027	*/
12028	if (fStepping)
12029	{
12030	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12031	AssertRC(rc);
12032	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12033	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12034	{
12035	rcStrict = VINF_EM_DBG_STEPPED;
12036	break;
12037	}
12038	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12039	}
12040
12041	/*
12042	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12043	*/
12044	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12045	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12046	}
12047
12048	/*
12049	* Clear the X86_EFL_TF if necessary.
12050	*/
12051	if (pVCpu->hm.s.fClearTrapFlag)
12052	{
12053	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12054	AssertRC(rc);
12055	pVCpu->hm.s.fClearTrapFlag = false;
12056	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12057	}
12058	/** @todo there seems to be issues with the resume flag when the monitor trap
12059	* flag is pending without being used. Seen early in bios init when
12060	* accessing APIC page in protected mode. */
12061
12062	/*
12063	* Restore VM-exit control settings as we may not re-enter this function the
12064	* next time around.
12065	*/
12066	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12067
12068	/* Restore HMCPU indicators. */
12069	pVCpu->hm.s.fUsingDebugLoop = false;
12070	pVCpu->hm.s.fDebugWantRdTscExit = false;
12071	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12072
12073	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12074	return rcStrict;
12075	}
12076
12077
12078	/** @} */
12079
12080
12081	/**
12082	* Checks if any expensive dtrace probes are enabled and we should go to the
12083	* debug loop.
12084	*
12085	* @returns true if we should use debug loop, false if not.
12086	*/
12087	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12088	{
12089	/* It's probably faster to OR the raw 32-bit counter variables together.
12090	Since the variables are in an array and the probes are next to one
12091	another (more or less), we have good locality. So, better read
12092	eight-nine cache lines ever time and only have one conditional, than
12093	128+ conditionals, right? */
12094	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12095	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12096	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12097	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12098	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12099	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12100	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12101	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12102	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12103	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12104	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12105	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12106	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12107	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12108	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12109	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12110	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12111	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12112	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12113	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12114	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12115	) != 0
12116	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12117	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12118	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12119	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12120	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12121	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12122	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12123	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12124	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12125	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12126	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12127	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12128	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12129	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12130	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12131	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12132	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12133	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12134	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12135	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12136	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12137	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12138	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12139	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12140	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12141	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12142	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12143	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12144	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12145	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12146	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12147	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12148	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12149	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12150	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12151	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12152	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12153	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12154	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12155	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12156	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12157	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12158	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12159	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12160	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12161	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12162	) != 0
12163	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12164	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12165	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12166	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12167	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12168	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12169	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12170	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12171	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12172	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12173	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12174	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12175	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12176	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12177	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12178	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12179	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12180	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12181	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12182	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12183	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12184	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12185	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12186	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12187	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12188	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12189	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12190	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12191	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12192	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12193	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12194	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12195	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12196	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12197	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12198	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12199	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12200	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12201	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12202	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12203	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12204	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12205	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12206	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12207	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12208	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12209	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12210	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12211	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12212	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12213	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12214	) != 0;
12215	}
12216
12217
12218	/**
12219	* Runs the guest using hardware-assisted VMX.
12220	*
12221	* @returns Strict VBox status code (i.e. informational status codes too).
12222	* @param pVCpu The cross context virtual CPU structure.
12223	*/
12224	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
12225	{
12226	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12227	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12228	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12229	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12230
12231	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
12232
12233	VBOXSTRICTRC rcStrict;
12234	uint32_t cLoops = 0;
12235	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12236	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12237	#else
12238	bool const fInNestedGuestMode = false;
12239	#endif
12240	if (!fInNestedGuestMode)
12241	{
12242	if ( !pVCpu->hm.s.fUseDebugLoop
12243	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12244	&& !DBGFIsStepping(pVCpu)
12245	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12246	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12247	else
12248	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12249	}
12250	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12251	else
12252	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
12253
12254	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12255	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12256	#endif
12257
12258	if (rcStrict == VERR_EM_INTERPRETER)
12259	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12260	else if (rcStrict == VINF_EM_RESET)
12261	rcStrict = VINF_EM_TRIPLE_FAULT;
12262
12263	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12264	if (RT_FAILURE(rc2))
12265	{
12266	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12267	rcStrict = rc2;
12268	}
12269	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12270	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12271	return rcStrict;
12272	}
12273
12274
12275	#ifndef HMVMX_USE_FUNCTION_TABLE
12276	/**
12277	* Handles a guest VM-exit from hardware-assisted VMX execution.
12278	*
12279	* @returns Strict VBox status code (i.e. informational status codes too).
12280	* @param pVCpu The cross context virtual CPU structure.
12281	* @param pVmxTransient The VMX-transient structure.
12282	*/
12283	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12284	{
12285	#ifdef DEBUG_ramshankar
12286	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12287	do { \
12288	if (a_fSave != 0) \
12289	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
12290	VBOXSTRICTRC rcStrict = a_CallExpr; \
12291	if (a_fSave != 0) \
12292	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12293	return rcStrict; \
12294	} while (0)
12295	#else
12296	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12297	#endif
12298	uint32_t const rcReason = pVmxTransient->uExitReason;
12299	switch (rcReason)
12300	{
12301	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12302	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12303	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12304	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12305	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12306	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12307	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12308	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12309	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12310	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12311	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12312	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12313	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12314	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12315	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12316	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12317	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12318	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12319	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12320	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12321	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12322	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12323	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12324	case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
12325	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12326	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12327	case VMX_EXIT_GDTR_IDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12328	case VMX_EXIT_LDTR_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12329	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12330	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12331	case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
12332	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12333	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12334	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12335	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12336	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12337	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12338	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12339	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12340	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12341	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12342	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12343	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12344	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12345	#else
12346	case VMX_EXIT_VMCLEAR:
12347	case VMX_EXIT_VMLAUNCH:
12348	case VMX_EXIT_VMPTRLD:
12349	case VMX_EXIT_VMPTRST:
12350	case VMX_EXIT_VMREAD:
12351	case VMX_EXIT_VMRESUME:
12352	case VMX_EXIT_VMWRITE:
12353	case VMX_EXIT_VMXOFF:
12354	case VMX_EXIT_VMXON:
12355	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12356	#endif
12357
12358	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12359	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12360	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
12361	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
12362	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
12363	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
12364	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
12365	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12366	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
12367
12368	case VMX_EXIT_INVEPT:
12369	case VMX_EXIT_INVVPID:
12370	case VMX_EXIT_VMFUNC:
12371	case VMX_EXIT_XSAVES:
12372	case VMX_EXIT_XRSTORS:
12373	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12374
12375	case VMX_EXIT_ENCLS:
12376	case VMX_EXIT_RDSEED:
12377	case VMX_EXIT_PML_FULL:
12378	default:
12379	return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
12380	}
12381	#undef VMEXIT_CALL_RET
12382	}
12383	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12384
12385
12386	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12387	/**
12388	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12389	*
12390	* @returns Strict VBox status code (i.e. informational status codes too).
12391	* @param pVCpu The cross context virtual CPU structure.
12392	* @param pVmxTransient The VMX-transient structure.
12393	*/
12394	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12395	{
12396	uint32_t const rcReason = pVmxTransient->uExitReason;
12397	switch (rcReason)
12398	{
12399	case VMX_EXIT_EPT_MISCONFIG:
12400	case VMX_EXIT_EPT_VIOLATION:
12401	case VMX_EXIT_IO_INSTR:
12402	case VMX_EXIT_CPUID:
12403	case VMX_EXIT_RDTSC:
12404	case VMX_EXIT_RDTSCP:
12405	case VMX_EXIT_APIC_ACCESS:
12406	case VMX_EXIT_XCPT_OR_NMI:
12407	case VMX_EXIT_MOV_CRX:
12408	case VMX_EXIT_EXT_INT:
12409	case VMX_EXIT_INT_WINDOW:
12410	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12411	case VMX_EXIT_MWAIT:
12412	case VMX_EXIT_MONITOR:
12413	case VMX_EXIT_TASK_SWITCH:
12414	case VMX_EXIT_PREEMPT_TIMER:
12415	case VMX_EXIT_RDMSR:
12416	case VMX_EXIT_WRMSR:
12417	case VMX_EXIT_VMCALL:
12418	case VMX_EXIT_MOV_DRX:
12419	case VMX_EXIT_HLT:
12420	case VMX_EXIT_INVD:
12421	case VMX_EXIT_INVLPG:
12422	case VMX_EXIT_RSM:
12423	case VMX_EXIT_MTF:
12424	case VMX_EXIT_PAUSE:
12425	case VMX_EXIT_GDTR_IDTR_ACCESS:
12426	case VMX_EXIT_LDTR_TR_ACCESS:
12427	case VMX_EXIT_WBINVD:
12428	case VMX_EXIT_XSETBV:
12429	case VMX_EXIT_RDRAND:
12430	case VMX_EXIT_INVPCID:
12431	case VMX_EXIT_GETSEC:
12432	case VMX_EXIT_RDPMC:
12433	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12434	case VMX_EXIT_VMCLEAR:
12435	case VMX_EXIT_VMLAUNCH:
12436	case VMX_EXIT_VMPTRLD:
12437	case VMX_EXIT_VMPTRST:
12438	case VMX_EXIT_VMREAD:
12439	case VMX_EXIT_VMRESUME:
12440	case VMX_EXIT_VMWRITE:
12441	case VMX_EXIT_VMXOFF:
12442	case VMX_EXIT_VMXON:
12443	#endif
12444	case VMX_EXIT_TRIPLE_FAULT:
12445	case VMX_EXIT_NMI_WINDOW:
12446	case VMX_EXIT_INIT_SIGNAL:
12447	case VMX_EXIT_SIPI:
12448	case VMX_EXIT_IO_SMI:
12449	case VMX_EXIT_SMI:
12450	case VMX_EXIT_ERR_MSR_LOAD:
12451	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12452	case VMX_EXIT_ERR_MACHINE_CHECK:
12453
12454	case VMX_EXIT_INVEPT:
12455	case VMX_EXIT_INVVPID:
12456	case VMX_EXIT_VMFUNC:
12457	case VMX_EXIT_XSAVES:
12458	case VMX_EXIT_XRSTORS:
12459
12460	case VMX_EXIT_ENCLS:
12461	case VMX_EXIT_RDSEED:
12462	case VMX_EXIT_PML_FULL:
12463	default:
12464	return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
12465	}
12466	#undef VMEXIT_CALL_RET
12467	}
12468	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12469
12470
12471	#ifdef VBOX_STRICT
12472	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
12473	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
12474	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
12475
12476	# define HMVMX_ASSERT_PREEMPT_CPUID() \
12477	do { \
12478	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
12479	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
12480	} while (0)
12481
12482	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12483	do { \
12484	AssertPtr((a_pVCpu)); \
12485	AssertPtr((a_pVmxTransient)); \
12486	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
12487	Assert((a_pVmxTransient)->pVmcsInfo); \
12488	Assert(ASMIntAreEnabled()); \
12489	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12490	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
12491	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
12492	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12493	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
12494	HMVMX_ASSERT_PREEMPT_CPUID(); \
12495	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12496	} while (0)
12497
12498	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12499	do { \
12500	Log4Func(("\n")); \
12501	} while (0)
12502	#else
12503	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12504	do { \
12505	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12506	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
12507	} while (0)
12508	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
12509	#endif
12510
12511
12512	/**
12513	* Advances the guest RIP by the specified number of bytes.
12514	*
12515	* @param pVCpu The cross context virtual CPU structure.
12516	* @param cbInstr Number of bytes to advance the RIP by.
12517	*
12518	* @remarks No-long-jump zone!!!
12519	*/
12520	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
12521	{
12522	/* Advance the RIP. */
12523	pVCpu->cpum.GstCtx.rip += cbInstr;
12524	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12525
12526	/* Update interrupt inhibition. */
12527	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
12528	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
12529	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12530	}
12531
12532
12533	/**
12534	* Advances the guest RIP after reading it from the VMCS.
12535	*
12536	* @returns VBox status code, no informational status codes.
12537	* @param pVCpu The cross context virtual CPU structure.
12538	* @param pVmxTransient The VMX-transient structure.
12539	*
12540	* @remarks No-long-jump zone!!!
12541	*/
12542	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12543	{
12544	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12545	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
12546	AssertRCReturn(rc, rc);
12547
12548	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
12549	return VINF_SUCCESS;
12550	}
12551
12552
12553	/**
12554	* Handle a condition that occurred while delivering an event through the guest
12555	* IDT.
12556	*
12557	* @returns Strict VBox status code (i.e. informational status codes too).
12558	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
12559	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
12560	* to continue execution of the guest which will delivery the \#DF.
12561	* @retval VINF_EM_RESET if we detected a triple-fault condition.
12562	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
12563	*
12564	* @param pVCpu The cross context virtual CPU structure.
12565	* @param pVmxTransient The VMX-transient structure.
12566	*
12567	* @remarks No-long-jump zone!!!
12568	*/
12569	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12570	{
12571	uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12572
12573	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12574	rc2 \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12575	AssertRCReturn(rc2, rc2);
12576
12577	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
12578	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12579	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12580	{
12581	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12582	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12583
12584	/*
12585	* If the event was a software interrupt (generated with INT n) or a software exception
12586	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
12587	* can handle the VM-exit and continue guest execution which will re-execute the
12588	* instruction rather than re-injecting the exception, as that can cause premature
12589	* trips to ring-3 before injection and involve TRPM which currently has no way of
12590	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
12591	* the problem).
12592	*/
12593	IEMXCPTRAISE enmRaise;
12594	IEMXCPTRAISEINFO fRaiseInfo;
12595	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
12596	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
12597	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
12598	{
12599	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
12600	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12601	}
12602	else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
12603	{
12604	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12605	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
12606	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
12607	/** @todo Make AssertMsgReturn as just AssertMsg later. */
12608	AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
12609	("Unexpected VM-exit interruption vector type %#x!\n", uExitVectorType), VERR_VMX_IPE_5);
12610
12611	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
12612
12613	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
12614	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
12615	{
12616	pVmxTransient->fVectoringPF = true;
12617	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12618	}
12619	}
12620	else
12621	{
12622	/*
12623	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
12624	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
12625	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
12626	*/
12627	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12628	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12629	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
12630	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12631	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12632	}
12633
12634	/*
12635	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
12636	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
12637	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
12638	* subsequent VM-entry would fail.
12639	*
12640	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
12641	*/
12642	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS)
12643	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12644	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
12645	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
12646	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
12647	{
12648	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
12649	}
12650
12651	switch (enmRaise)
12652	{
12653	case IEMXCPTRAISE_CURRENT_XCPT:
12654	{
12655	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
12656	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
12657	Assert(rcStrict == VINF_SUCCESS);
12658	break;
12659	}
12660
12661	case IEMXCPTRAISE_PREV_EVENT:
12662	{
12663	uint32_t u32ErrCode;
12664	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
12665	{
12666	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12667	AssertRCReturn(rc2, rc2);
12668	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12669	}
12670	else
12671	u32ErrCode = 0;
12672
12673	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
12674	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
12675	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12676	0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
12677
12678	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
12679	pVCpu->hm.s.Event.u32ErrCode));
12680	Assert(rcStrict == VINF_SUCCESS);
12681	break;
12682	}
12683
12684	case IEMXCPTRAISE_REEXEC_INSTR:
12685	Assert(rcStrict == VINF_SUCCESS);
12686	break;
12687
12688	case IEMXCPTRAISE_DOUBLE_FAULT:
12689	{
12690	/*
12691	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
12692	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
12693	*/
12694	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
12695	{
12696	pVmxTransient->fVectoringDoublePF = true;
12697	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
12698	pVCpu->cpum.GstCtx.cr2));
12699	rcStrict = VINF_SUCCESS;
12700	}
12701	else
12702	{
12703	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
12704	hmR0VmxSetPendingXcptDF(pVCpu);
12705	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
12706	uIdtVector, uExitVector));
12707	rcStrict = VINF_HM_DOUBLE_FAULT;
12708	}
12709	break;
12710	}
12711
12712	case IEMXCPTRAISE_TRIPLE_FAULT:
12713	{
12714	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
12715	rcStrict = VINF_EM_RESET;
12716	break;
12717	}
12718
12719	case IEMXCPTRAISE_CPU_HANG:
12720	{
12721	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
12722	rcStrict = VERR_EM_GUEST_CPU_HANG;
12723	break;
12724	}
12725
12726	default:
12727	{
12728	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
12729	rcStrict = VERR_VMX_IPE_2;
12730	break;
12731	}
12732	}
12733	}
12734	else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
12735	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
12736	&& uExitVector != X86_XCPT_DF
12737	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
12738	{
12739	/*
12740	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
12741	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
12742	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
12743	*/
12744	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
12745	{
12746	Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
12747	VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
12748	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
12749	}
12750	}
12751
12752	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
12753	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
12754	return rcStrict;
12755	}
12756
12757
12758	/** @name VM-exit handlers.
12759	* @{
12760	*/
12761	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12762	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12763	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12764
12765	/**
12766	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
12767	*/
12768	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12769	{
12770	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12771	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
12772	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
12773	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
12774	return VINF_SUCCESS;
12775	return VINF_EM_RAW_INTERRUPT;
12776	}
12777
12778
12779	/**
12780	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
12781	*/
12782	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12783	{
12784	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12785	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
12786
12787	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12788	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12789	AssertRCReturn(rc, rc);
12790
12791	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12792	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
12793	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
12794	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
12795
12796	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12797	{
12798	/*
12799	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
12800	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
12801	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
12802	*
12803	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
12804	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
12805	*/
12806	VMXDispatchHostNmi();
12807	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
12808	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12809	return VINF_SUCCESS;
12810	}
12811
12812	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12813	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12814	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
12815	{ /* likely */ }
12816	else
12817	{
12818	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
12819	rcStrictRc1 = VINF_SUCCESS;
12820	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12821	return rcStrictRc1;
12822	}
12823
12824	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
12825	uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
12826	switch (uIntType)
12827	{
12828	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
12829	Assert(uVector == X86_XCPT_DB);
12830	RT_FALL_THRU();
12831	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
12832	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
12833	RT_FALL_THRU();
12834	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
12835	{
12836	/*
12837	* If there's any exception caused as a result of event injection, the resulting
12838	* secondary/final execption will be pending, we shall continue guest execution
12839	* after injecting the event. The page-fault case is complicated and we manually
12840	* handle any currently pending event in hmR0VmxExitXcptPF.
12841	*/
12842	if (!pVCpu->hm.s.Event.fPending)
12843	{ /* likely */ }
12844	else if (uVector != X86_XCPT_PF)
12845	{
12846	rc = VINF_SUCCESS;
12847	break;
12848	}
12849
12850	switch (uVector)
12851	{
12852	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
12853	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
12854	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
12855	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
12856	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
12857	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
12858
12859	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
12860	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12861	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
12862	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12863	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
12864	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12865	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
12866	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12867	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
12868	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12869	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
12870	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12871	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
12872	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12873	default:
12874	{
12875	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
12876	if (pVmcsInfo->RealMode.fRealOnV86Active)
12877	{
12878	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
12879	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
12880	Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
12881
12882	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
12883	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12884	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12885	AssertRCReturn(rc, rc);
12886	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
12887	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
12888	0 /* GCPtrFaultAddress */);
12889	}
12890	else
12891	{
12892	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
12893	pVCpu->hm.s.u32HMError = uVector;
12894	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
12895	}
12896	break;
12897	}
12898	}
12899	break;
12900	}
12901
12902	default:
12903	{
12904	pVCpu->hm.s.u32HMError = uExitIntInfo;
12905	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
12906	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
12907	break;
12908	}
12909	}
12910	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12911	return rc;
12912	}
12913
12914
12915	/**
12916	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
12917	*/
12918	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12919	{
12920	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12921
12922	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
12923	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12924	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
12925	AssertRCReturn(rc, rc);
12926
12927	/* Evaluate and deliver pending events and resume guest execution. */
12928	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
12929	return VINF_SUCCESS;
12930	}
12931
12932
12933	/**
12934	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
12935	*/
12936	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12937	{
12938	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12939
12940	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12941	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
12942	{
12943	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
12944	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12945	}
12946
12947	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS));
12948
12949	/*
12950	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
12951	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
12952	*/
12953	uint32_t fIntrState;
12954	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
12955	AssertRCReturn(rc, rc);
12956	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
12957	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
12958	{
12959	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
12960	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12961
12962	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
12963	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
12964	AssertRCReturn(rc, rc);
12965	}
12966
12967	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
12968	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
12969	AssertRCReturn(rc, rc);
12970
12971	/* Evaluate and deliver pending events and resume guest execution. */
12972	return VINF_SUCCESS;
12973	}
12974
12975
12976	/**
12977	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
12978	*/
12979	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12980	{
12981	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12982	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12983	}
12984
12985
12986	/**
12987	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
12988	*/
12989	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12990	{
12991	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12992	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12993	}
12994
12995
12996	/**
12997	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
12998	*/
12999	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13000	{
13001	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13002
13003	/*
13004	* Get the state we need and update the exit history entry.
13005	*/
13006	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13007	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13008	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13009	AssertRCReturn(rc, rc);
13010
13011	VBOXSTRICTRC rcStrict;
13012	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
13013	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
13014	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
13015	if (!pExitRec)
13016	{
13017	/*
13018	* Regular CPUID instruction execution.
13019	*/
13020	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
13021	if (rcStrict == VINF_SUCCESS)
13022	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13023	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13024	{
13025	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13026	rcStrict = VINF_SUCCESS;
13027	}
13028	}
13029	else
13030	{
13031	/*
13032	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
13033	*/
13034	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13035	AssertRCReturn(rc2, rc2);
13036
13037	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
13038	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
13039
13040	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
13041	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13042
13043	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
13044	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13045	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
13046	}
13047	return rcStrict;
13048	}
13049
13050
13051	/**
13052	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
13053	*/
13054	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13055	{
13056	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13057
13058	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13059	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
13060	AssertRCReturn(rc, rc);
13061
13062	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
13063	return VINF_EM_RAW_EMULATE_INSTR;
13064
13065	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
13066	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13067	}
13068
13069
13070	/**
13071	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
13072	*/
13073	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13074	{
13075	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13076
13077	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13078	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13079	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13080	AssertRCReturn(rc, rc);
13081
13082	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
13083	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13084	{
13085	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13086	we must reset offsetting on VM-entry. See @bugref{6634}. */
13087	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13088	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13089	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13090	}
13091	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13092	{
13093	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13094	rcStrict = VINF_SUCCESS;
13095	}
13096	return rcStrict;
13097	}
13098
13099
13100	/**
13101	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
13102	*/
13103	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13104	{
13105	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13106
13107	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13108	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
13109	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13110	AssertRCReturn(rc, rc);
13111
13112	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
13113	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13114	{
13115	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13116	we must reset offsetting on VM-reentry. See @bugref{6634}. */
13117	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13118	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13119	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13120	}
13121	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13122	{
13123	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13124	rcStrict = VINF_SUCCESS;
13125	}
13126	return rcStrict;
13127	}
13128
13129
13130	/**
13131	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
13132	*/
13133	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13134	{
13135	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13136
13137	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13138	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
13139	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13140	AssertRCReturn(rc, rc);
13141
13142	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13143	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13144	if (RT_LIKELY(rc == VINF_SUCCESS))
13145	{
13146	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13147	Assert(pVmxTransient->cbInstr == 2);
13148	}
13149	else
13150	{
13151	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
13152	rc = VERR_EM_INTERPRETER;
13153	}
13154	return rc;
13155	}
13156
13157
13158	/**
13159	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
13160	*/
13161	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13162	{
13163	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13164
13165	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
13166	if (EMAreHypercallInstructionsEnabled(pVCpu))
13167	{
13168	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13169	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
13170	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13171	AssertRCReturn(rc, rc);
13172
13173	/* Perform the hypercall. */
13174	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
13175	if (rcStrict == VINF_SUCCESS)
13176	{
13177	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13178	AssertRCReturn(rc, rc);
13179	}
13180	else
13181	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
13182	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
13183	\|\| RT_FAILURE(rcStrict));
13184
13185	/* If the hypercall changes anything other than guest's general-purpose registers,
13186	we would need to reload the guest changed bits here before VM-entry. */
13187	}
13188	else
13189	Log4Func(("Hypercalls not enabled\n"));
13190
13191	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
13192	if (RT_FAILURE(rcStrict))
13193	{
13194	hmR0VmxSetPendingXcptUD(pVCpu);
13195	rcStrict = VINF_SUCCESS;
13196	}
13197
13198	return rcStrict;
13199	}
13200
13201
13202	/**
13203	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
13204	*/
13205	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13206	{
13207	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13208	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
13209
13210	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13211	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13212	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13213	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13214	AssertRCReturn(rc, rc);
13215
13216	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
13217
13218	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
13219	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13220	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13221	{
13222	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13223	rcStrict = VINF_SUCCESS;
13224	}
13225	else
13226	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n", pVmxTransient->uExitQual,
13227	VBOXSTRICTRC_VAL(rcStrict)));
13228	return rcStrict;
13229	}
13230
13231
13232	/**
13233	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
13234	*/
13235	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13236	{
13237	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13238
13239	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13240	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13241	AssertRCReturn(rc, rc);
13242
13243	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13244	rc = EMInterpretMonitor(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13245	if (RT_LIKELY(rc == VINF_SUCCESS))
13246	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13247	else
13248	{
13249	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
13250	rc = VERR_EM_INTERPRETER;
13251	}
13252	return rc;
13253	}
13254
13255
13256	/**
13257	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
13258	*/
13259	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13260	{
13261	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13262
13263	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13264	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13265	AssertRCReturn(rc, rc);
13266
13267	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13268	VBOXSTRICTRC rc2 = EMInterpretMWait(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13269	rc = VBOXSTRICTRC_VAL(rc2);
13270	if (RT_LIKELY( rc == VINF_SUCCESS
13271	\|\| rc == VINF_EM_HALT))
13272	{
13273	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13274	AssertRCReturn(rc3, rc3);
13275
13276	if ( rc == VINF_EM_HALT
13277	&& EMMonitorWaitShouldContinue(pVCpu, pCtx))
13278	rc = VINF_SUCCESS;
13279	}
13280	else
13281	{
13282	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
13283	rc = VERR_EM_INTERPRETER;
13284	}
13285	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
13286	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
13287	return rc;
13288	}
13289
13290
13291	/**
13292	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
13293	*/
13294	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13295	{
13296	/*
13297	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
13298	* mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
13299	* treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
13300	* VMX root operation. If we get here, something funny is going on.
13301	*
13302	* See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
13303	*/
13304	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13305	AssertMsgFailed(("Unexpected RSM VM-exit\n"));
13306	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13307	}
13308
13309
13310	/**
13311	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
13312	*/
13313	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13314	{
13315	/*
13316	* This can only happen if we support dual-monitor treatment of SMI, which can be activated
13317	* by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
13318	* this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
13319	* an SMI. If we get here, something funny is going on.
13320	*
13321	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
13322	* See Intel spec. 25.3 "Other Causes of VM-Exits"
13323	*/
13324	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13325	AssertMsgFailed(("Unexpected SMI VM-exit\n"));
13326	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13327	}
13328
13329
13330	/**
13331	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
13332	*/
13333	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13334	{
13335	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
13336	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13337	AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
13338	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13339	}
13340
13341
13342	/**
13343	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
13344	*/
13345	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13346	{
13347	/*
13348	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
13349	* We don't make use of it as our guests don't have direct access to the host LAPIC.
13350	* See Intel spec. 25.3 "Other Causes of VM-exits".
13351	*/
13352	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13353	AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
13354	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13355	}
13356
13357
13358	/**
13359	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
13360	* VM-exit.
13361	*/
13362	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13363	{
13364	/*
13365	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
13366	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
13367	*
13368	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
13369	* See Intel spec. "23.8 Restrictions on VMX operation".
13370	*/
13371	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13372	return VINF_SUCCESS;
13373	}
13374
13375
13376	/**
13377	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
13378	* VM-exit.
13379	*/
13380	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13381	{
13382	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13383	return VINF_EM_RESET;
13384	}
13385
13386
13387	/**
13388	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
13389	*/
13390	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13391	{
13392	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13393
13394	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13395	AssertRCReturn(rc, rc);
13396
13397	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
13398	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
13399	rc = VINF_SUCCESS;
13400	else
13401	rc = VINF_EM_HALT;
13402
13403	if (rc != VINF_SUCCESS)
13404	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
13405	return rc;
13406	}
13407
13408
13409	/**
13410	* VM-exit handler for instructions that result in a \#UD exception delivered to
13411	* the guest.
13412	*/
13413	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13414	{
13415	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13416	hmR0VmxSetPendingXcptUD(pVCpu);
13417	return VINF_SUCCESS;
13418	}
13419
13420
13421	/**
13422	* VM-exit handler for expiry of the VMX-preemption timer.
13423	*/
13424	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13425	{
13426	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13427
13428	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
13429	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13430
13431	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
13432	PVM pVM = pVCpu->CTX_SUFF(pVM);
13433	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
13434	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
13435	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
13436	}
13437
13438
13439	/**
13440	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
13441	*/
13442	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13443	{
13444	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13445
13446	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13447	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13448	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
13449	AssertRCReturn(rc, rc);
13450
13451	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
13452	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13453	: HM_CHANGED_RAISED_XCPT_MASK);
13454
13455	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13456	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
13457
13458	return rcStrict;
13459	}
13460
13461
13462	/**
13463	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
13464	*/
13465	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13466	{
13467	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13468	/** @todo Use VM-exit instruction information. */
13469	return VERR_EM_INTERPRETER;
13470	}
13471
13472
13473	/**
13474	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
13475	* Error VM-exit.
13476	*/
13477	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13478	{
13479	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13480	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13481	AssertRCReturn(rc, rc);
13482
13483	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo);
13484	if (RT_FAILURE(rc))
13485	return rc;
13486
13487	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
13488	NOREF(uInvalidReason);
13489
13490	#ifdef VBOX_STRICT
13491	uint32_t fIntrState;
13492	RTHCUINTREG uHCReg;
13493	uint64_t u64Val;
13494	uint32_t u32Val;
13495	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
13496	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
13497	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
13498	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13499	AssertRCReturn(rc, rc);
13500
13501	Log4(("uInvalidReason %u\n", uInvalidReason));
13502	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
13503	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
13504	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
13505	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
13506
13507	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
13508	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
13509	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
13510	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
13511	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
13512	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13513	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
13514	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
13515	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
13516	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13517	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
13518	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
13519
13520	hmR0DumpRegs(pVCpu);
13521	#endif
13522
13523	return VERR_VMX_INVALID_GUEST_STATE;
13524	}
13525
13526
13527	/**
13528	* VM-exit handler for VM-entry failure due to an MSR-load
13529	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
13530	*/
13531	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13532	{
13533	AssertMsgFailed(("Unexpected MSR-load exit\n"));
13534	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13535	}
13536
13537
13538	/**
13539	* VM-exit handler for VM-entry failure due to a machine-check event
13540	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
13541	*/
13542	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13543	{
13544	AssertMsgFailed(("Unexpected machine-check event exit\n"));
13545	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13546	}
13547
13548
13549	/**
13550	* VM-exit handler for all undefined reasons. Should never ever happen.. in
13551	* theory.
13552	*/
13553	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13554	{
13555	RT_NOREF2(pVCpu, pVmxTransient);
13556	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
13557	return VERR_VMX_UNDEFINED_EXIT_CODE;
13558	}
13559
13560
13561	/**
13562	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
13563	* (VMX_EXIT_GDTR_IDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
13564	* Conditional VM-exit.
13565	*/
13566	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13567	{
13568	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13569
13570	/* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
13571	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
13572	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13573	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
13574	return VERR_EM_INTERPRETER;
13575	AssertMsgFailed(("Unexpected XDTR access\n"));
13576	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13577	}
13578
13579
13580	/**
13581	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
13582	*/
13583	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13584	{
13585	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13586
13587	/* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
13588	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13589	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
13590	return VERR_EM_INTERPRETER;
13591	AssertMsgFailed(("Unexpected RDRAND exit\n"));
13592	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13593	}
13594
13595
13596	/**
13597	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
13598	*/
13599	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13600	{
13601	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13602
13603	/** @todo Optimize this: We currently drag in in the whole MSR state
13604	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13605	* MSRs required. That would require changes to IEM and possibly CPUM too.
13606	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13607	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13608	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13609	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13610	switch (idMsr)
13611	{
13612	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13613	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13614	}
13615
13616	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13617	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13618	AssertRCReturn(rc, rc);
13619
13620	Log4Func(("ecx=%#RX32\n", idMsr));
13621
13622	#ifdef VBOX_STRICT
13623	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
13624	{
13625	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
13626	&& idMsr != MSR_K6_EFER)
13627	{
13628	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
13629	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13630	}
13631	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13632	{
13633	Assert(pVmcsInfo->pvMsrBitmap);
13634	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13635	if (fMsrpm & VMXMSRPM_ALLOW_RD)
13636	{
13637	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
13638	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13639	}
13640	}
13641	}
13642	#endif
13643
13644	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
13645	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
13646	if (rcStrict == VINF_SUCCESS)
13647	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13648	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
13649	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13650	{
13651	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13652	rcStrict = VINF_SUCCESS;
13653	}
13654	else
13655	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13656
13657	return rcStrict;
13658	}
13659
13660
13661	/**
13662	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
13663	*/
13664	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13665	{
13666	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13667
13668	/** @todo Optimize this: We currently drag in in the whole MSR state
13669	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13670	* MSRs required. That would require changes to IEM and possibly CPUM too.
13671	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13672	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13673	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13674
13675	/*
13676	* The FS and GS base MSRs are not part of the above all-MSRs mask.
13677	* Although we don't need to fetch the base as it will be overwritten shortly, while
13678	* loading guest-state we would also load the entire segment register including limit
13679	* and attributes and thus we need to load them here.
13680	*/
13681	switch (idMsr)
13682	{
13683	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13684	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13685	}
13686
13687	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13688	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13689	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13690	AssertRCReturn(rc, rc);
13691
13692	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
13693
13694	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
13695	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
13696
13697	if (rcStrict == VINF_SUCCESS)
13698	{
13699	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13700
13701	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
13702	if ( idMsr == MSR_IA32_APICBASE
13703	\|\| ( idMsr >= MSR_IA32_X2APIC_START
13704	&& idMsr <= MSR_IA32_X2APIC_END))
13705	{
13706	/*
13707	* We've already saved the APIC related guest-state (TPR) in post-run phase.
13708	* When full APIC register virtualization is implemented we'll have to make
13709	* sure APIC state is saved from the VMCS before IEM changes it.
13710	*/
13711	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
13712	}
13713	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
13714	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13715	else if (idMsr == MSR_K6_EFER)
13716	{
13717	/*
13718	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
13719	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
13720	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
13721	*/
13722	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
13723	}
13724
13725	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
13726	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
13727	{
13728	switch (idMsr)
13729	{
13730	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
13731	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
13732	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
13733	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
13734	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
13735	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
13736	default:
13737	{
13738	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
13739	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
13740	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13741	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
13742	break;
13743	}
13744	}
13745	}
13746	#ifdef VBOX_STRICT
13747	else
13748	{
13749	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
13750	switch (idMsr)
13751	{
13752	case MSR_IA32_SYSENTER_CS:
13753	case MSR_IA32_SYSENTER_EIP:
13754	case MSR_IA32_SYSENTER_ESP:
13755	case MSR_K8_FS_BASE:
13756	case MSR_K8_GS_BASE:
13757	{
13758	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
13759	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13760	}
13761
13762	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
13763	default:
13764	{
13765	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
13766	{
13767	/* EFER MSR writes are always intercepted. */
13768	if (idMsr != MSR_K6_EFER)
13769	{
13770	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
13771	idMsr));
13772	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13773	}
13774	}
13775
13776	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13777	{
13778	Assert(pVmcsInfo->pvMsrBitmap);
13779	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13780	if (fMsrpm & VMXMSRPM_ALLOW_WR)
13781	{
13782	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
13783	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13784	}
13785	}
13786	break;
13787	}
13788	}
13789	}
13790	#endif /* VBOX_STRICT */
13791	}
13792	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13793	{
13794	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13795	rcStrict = VINF_SUCCESS;
13796	}
13797	else
13798	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13799
13800	return rcStrict;
13801	}
13802
13803
13804	/**
13805	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
13806	*/
13807	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13808	{
13809	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13810	/** @todo The guest has likely hit a contended spinlock. We might want to
13811	* poke a schedule different guest VCPU. */
13812	return VINF_EM_RAW_INTERRUPT;
13813	}
13814
13815
13816	/**
13817	* VM-exit handler for when the TPR value is lowered below the specified
13818	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
13819	*/
13820	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13821	{
13822	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13823	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
13824
13825	/*
13826	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
13827	* We'll re-evaluate pending interrupts and inject them before the next VM
13828	* entry so we can just continue execution here.
13829	*/
13830	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
13831	return VINF_SUCCESS;
13832	}
13833
13834
13835	/**
13836	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
13837	* VM-exit.
13838	*
13839	* @retval VINF_SUCCESS when guest execution can continue.
13840	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
13841	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
13842	* interpreter.
13843	*/
13844	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13845	{
13846	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13847	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
13848
13849	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13850	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13851	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13852	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13853	AssertRCReturn(rc, rc);
13854
13855	VBOXSTRICTRC rcStrict;
13856	PVM pVM = pVCpu->CTX_SUFF(pVM);
13857	RTGCUINTPTR const uExitQual = pVmxTransient->uExitQual;
13858	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
13859	switch (uAccessType)
13860	{
13861	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
13862	{
13863	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
13864	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
13865	VMX_EXIT_QUAL_CRX_GENREG(uExitQual));
13866	AssertMsg( rcStrict == VINF_SUCCESS
13867	\|\| rcStrict == VINF_IEM_RAISED_XCPT
13868	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13869
13870	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
13871	{
13872	case 0:
13873	{
13874	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13875	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13876	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
13877	Log4Func(("CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
13878
13879	/*
13880	* This is a kludge for handling switches back to real mode when we try to use
13881	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
13882	* deal with special selector values, so we have to return to ring-3 and run
13883	* there till the selector values are V86 mode compatible.
13884	*
13885	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
13886	* latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
13887	* at the end of this function.
13888	*/
13889	if ( rc == VINF_SUCCESS
13890	&& !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
13891	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
13892	&& (uOldCr0 & X86_CR0_PE)
13893	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
13894	{
13895	/** @todo check selectors rather than returning all the time. */
13896	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
13897	rcStrict = VINF_EM_RESCHEDULE_REM;
13898	}
13899	break;
13900	}
13901
13902	case 2:
13903	{
13904	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
13905	/* Nothing to do here, CR2 it's not part of the VMCS. */
13906	break;
13907	}
13908
13909	case 3:
13910	{
13911	Assert( !pVM->hm.s.fNestedPaging
13912	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
13913	\|\| pVCpu->hm.s.fUsingDebugLoop);
13914	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
13915	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13916	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
13917	Log4Func(("CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
13918	break;
13919	}
13920
13921	case 4:
13922	{
13923	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
13924	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13925	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
13926	Log4Func(("CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
13927	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
13928	break;
13929	}
13930
13931	case 8:
13932	{
13933	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
13934	Assert(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
13935	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13936	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
13937	break;
13938	}
13939	default:
13940	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual)));
13941	break;
13942	}
13943	break;
13944	}
13945
13946	case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
13947	{
13948	Assert( !pVM->hm.s.fNestedPaging
13949	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
13950	\|\| pVCpu->hm.s.fUsingDebugLoop
13951	\|\| VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 3);
13952	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
13953	Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 8
13954	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
13955
13956	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_GENREG(uExitQual),
13957	VMX_EXIT_QUAL_CRX_REGISTER(uExitQual));
13958	AssertMsg( rcStrict == VINF_SUCCESS
13959	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13960	#ifdef VBOX_WITH_STATISTICS
13961	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
13962	{
13963	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
13964	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
13965	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
13966	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
13967	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
13968	}
13969	#endif
13970	Log4Func(("CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
13971	VBOXSTRICTRC_VAL(rcStrict)));
13972	if (VMX_EXIT_QUAL_CRX_GENREG(uExitQual) == X86_GREG_xSP)
13973	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
13974	else
13975	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13976	break;
13977	}
13978
13979	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
13980	{
13981	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
13982	AssertMsg( rcStrict == VINF_SUCCESS
13983	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13984
13985	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13986	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
13987	Log4Func(("CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
13988	break;
13989	}
13990
13991	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
13992	{
13993	/* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
13994	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
13995	AssertRCReturn(rc, rc);
13996	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual),
13997	pVmxTransient->uGuestLinearAddr);
13998	AssertMsg( rcStrict == VINF_SUCCESS
13999	\|\| rcStrict == VINF_IEM_RAISED_XCPT
14000	, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14001
14002	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14003	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
14004	Log4Func(("LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
14005	break;
14006	}
14007
14008	default:
14009	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
14010	VERR_VMX_UNEXPECTED_EXCEPTION);
14011	}
14012
14013	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
14014	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
14015	if (rcStrict == VINF_IEM_RAISED_XCPT)
14016	{
14017	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14018	rcStrict = VINF_SUCCESS;
14019	}
14020
14021	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
14022	NOREF(pVM);
14023	return rcStrict;
14024	}
14025
14026
14027	/**
14028	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
14029	* VM-exit.
14030	*/
14031	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14032	{
14033	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14034	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
14035
14036	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14037	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14038	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14039	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14040	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
14041	\| CPUMCTX_EXTRN_EFER);
14042	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
14043	AssertRCReturn(rc, rc);
14044
14045	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
14046	uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
14047	uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQual);
14048	bool fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
14049	bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
14050	bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
14051	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
14052	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
14053
14054	/*
14055	* Update exit history to see if this exit can be optimized.
14056	*/
14057	VBOXSTRICTRC rcStrict;
14058	PCEMEXITREC pExitRec = NULL;
14059	if ( !fGstStepping
14060	&& !fDbgStepping)
14061	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14062	!fIOString
14063	? !fIOWrite
14064	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
14065	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
14066	: !fIOWrite
14067	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
14068	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
14069	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14070	if (!pExitRec)
14071	{
14072	/* I/O operation lookup arrays. */
14073	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
14074	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
14075	uint32_t const cbValue = s_aIOSizes[uIOWidth];
14076	uint32_t const cbInstr = pVmxTransient->cbInstr;
14077	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
14078	PVM pVM = pVCpu->CTX_SUFF(pVM);
14079	if (fIOString)
14080	{
14081	/*
14082	* INS/OUTS - I/O String instruction.
14083	*
14084	* Use instruction-information if available, otherwise fall back on
14085	* interpreting the instruction.
14086	*/
14087	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14088	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
14089	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
14090	if (fInsOutsInfo)
14091	{
14092	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14093	AssertRCReturn(rc2, rc2);
14094	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
14095	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
14096	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
14097	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
14098	if (fIOWrite)
14099	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
14100	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
14101	else
14102	{
14103	/*
14104	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
14105	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
14106	* See Intel Instruction spec. for "INS".
14107	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
14108	*/
14109	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
14110	}
14111	}
14112	else
14113	rcStrict = IEMExecOne(pVCpu);
14114
14115	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14116	fUpdateRipAlready = true;
14117	}
14118	else
14119	{
14120	/*
14121	* IN/OUT - I/O instruction.
14122	*/
14123	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14124	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
14125	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
14126	if (fIOWrite)
14127	{
14128	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
14129	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
14130	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14131	&& !pCtx->eflags.Bits.u1TF)
14132	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
14133	}
14134	else
14135	{
14136	uint32_t u32Result = 0;
14137	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
14138	if (IOM_SUCCESS(rcStrict))
14139	{
14140	/* Save result of I/O IN instr. in AL/AX/EAX. */
14141	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
14142	}
14143	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14144	&& !pCtx->eflags.Bits.u1TF)
14145	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
14146	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
14147	}
14148	}
14149
14150	if (IOM_SUCCESS(rcStrict))
14151	{
14152	if (!fUpdateRipAlready)
14153	{
14154	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
14155	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14156	}
14157
14158	/*
14159	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
14160	* while booting Fedora 17 64-bit guest.
14161	*
14162	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
14163	*/
14164	if (fIOString)
14165	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
14166
14167	/*
14168	* If any I/O breakpoints are armed, we need to check if one triggered
14169	* and take appropriate action.
14170	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
14171	*/
14172	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
14173	AssertRCReturn(rc, rc);
14174
14175	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
14176	* execution engines about whether hyper BPs and such are pending. */
14177	uint32_t const uDr7 = pCtx->dr[7];
14178	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
14179	&& X86_DR7_ANY_RW_IO(uDr7)
14180	&& (pCtx->cr4 & X86_CR4_DE))
14181	\|\| DBGFBpIsHwIoArmed(pVM)))
14182	{
14183	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
14184
14185	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
14186	VMMRZCallRing3Disable(pVCpu);
14187	HM_DISABLE_PREEMPT(pVCpu);
14188
14189	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
14190
14191	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
14192	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
14193	{
14194	/* Raise #DB. */
14195	if (fIsGuestDbgActive)
14196	ASMSetDR6(pCtx->dr[6]);
14197	if (pCtx->dr[7] != uDr7)
14198	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
14199
14200	hmR0VmxSetPendingXcptDB(pVCpu);
14201	}
14202	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
14203	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
14204	else if ( rcStrict2 != VINF_SUCCESS
14205	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
14206	rcStrict = rcStrict2;
14207	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
14208
14209	HM_RESTORE_PREEMPT();
14210	VMMRZCallRing3Enable(pVCpu);
14211	}
14212	}
14213
14214	#ifdef VBOX_STRICT
14215	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14216	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
14217	Assert(!fIOWrite);
14218	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14219	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
14220	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
14221	Assert(fIOWrite);
14222	else
14223	{
14224	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
14225	* statuses, that the VMM device and some others may return. See
14226	* IOM_SUCCESS() for guidance. */
14227	AssertMsg( RT_FAILURE(rcStrict)
14228	\|\| rcStrict == VINF_SUCCESS
14229	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
14230	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
14231	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
14232	\|\| rcStrict == VINF_EM_RAW_TO_R3
14233	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14234	# endif
14235	}
14236	#endif
14237	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
14238	}
14239	else
14240	{
14241	/*
14242	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14243	*/
14244	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14245	AssertRCReturn(rc2, rc2);
14246	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
14247	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
14248	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
14249	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14250	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
14251	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
14252
14253	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14254	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14255
14256	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14257	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14258	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14259	}
14260	return rcStrict;
14261	}
14262
14263
14264	/**
14265	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
14266	* VM-exit.
14267	*/
14268	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14269	{
14270	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14271
14272	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
14273	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14274	AssertRCReturn(rc, rc);
14275	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
14276	{
14277	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14278	AssertRCReturn(rc, rc);
14279	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
14280	{
14281	uint32_t uErrCode;
14282	RTGCUINTPTR GCPtrFaultAddress;
14283	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
14284	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
14285	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
14286	if (fErrorCodeValid)
14287	{
14288	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14289	AssertRCReturn(rc, rc);
14290	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
14291	}
14292	else
14293	uErrCode = 0;
14294
14295	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
14296	&& uVector == X86_XCPT_PF)
14297	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
14298	else
14299	GCPtrFaultAddress = 0;
14300
14301	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14302	AssertRCReturn(rc, rc);
14303
14304	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
14305	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
14306
14307	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
14308	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14309	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14310	}
14311	}
14312
14313	/* Fall back to the interpreter to emulate the task-switch. */
14314	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14315	return VERR_EM_INTERPRETER;
14316	}
14317
14318
14319	/**
14320	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
14321	*/
14322	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14323	{
14324	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14325
14326	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14327	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
14328	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14329	AssertRCReturn(rc, rc);
14330	return VINF_EM_DBG_STEPPED;
14331	}
14332
14333
14334	/**
14335	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
14336	*/
14337	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14338	{
14339	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14340	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
14341
14342	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14343	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14344	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14345	{
14346	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
14347	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14348	{
14349	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14350	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14351	}
14352	}
14353	else
14354	{
14355	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14356	rcStrict1 = VINF_SUCCESS;
14357	return rcStrict1;
14358	}
14359
14360	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
14361	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14362	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14363	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14364	AssertRCReturn(rc, rc);
14365
14366	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
14367	uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
14368	VBOXSTRICTRC rcStrict2;
14369	switch (uAccessType)
14370	{
14371	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
14372	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
14373	{
14374	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
14375	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
14376	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
14377
14378	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
14379	GCPhys &= PAGE_BASE_GC_MASK;
14380	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
14381	PVM pVM = pVCpu->CTX_SUFF(pVM);
14382	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
14383	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
14384
14385	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14386	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
14387	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
14388	CPUMCTX2CORE(pCtx), GCPhys);
14389	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14390	if ( rcStrict2 == VINF_SUCCESS
14391	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14392	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14393	{
14394	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14395	\| HM_CHANGED_GUEST_APIC_TPR);
14396	rcStrict2 = VINF_SUCCESS;
14397	}
14398	break;
14399	}
14400
14401	default:
14402	Log4Func(("uAccessType=%#x\n", uAccessType));
14403	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
14404	break;
14405	}
14406
14407	if (rcStrict2 != VINF_SUCCESS)
14408	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
14409	return rcStrict2;
14410	}
14411
14412
14413	/**
14414	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
14415	* VM-exit.
14416	*/
14417	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14418	{
14419	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14420
14421	/* We should -not- get this VM-exit if the guest's debug registers were active. */
14422	if (pVmxTransient->fWasGuestDebugStateActive)
14423	{
14424	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
14425	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14426	}
14427
14428	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14429	if ( !pVCpu->hm.s.fSingleInstruction
14430	&& !pVmxTransient->fWasHyperDebugStateActive)
14431	{
14432	Assert(!DBGFIsStepping(pVCpu));
14433	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
14434
14435	/* Don't intercept MOV DRx any more. */
14436	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
14437	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14438	AssertRCReturn(rc, rc);
14439
14440	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
14441	VMMRZCallRing3Disable(pVCpu);
14442	HM_DISABLE_PREEMPT(pVCpu);
14443
14444	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
14445	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
14446	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
14447
14448	HM_RESTORE_PREEMPT();
14449	VMMRZCallRing3Enable(pVCpu);
14450
14451	#ifdef VBOX_WITH_STATISTICS
14452	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14453	AssertRCReturn(rc, rc);
14454	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14455	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14456	else
14457	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14458	#endif
14459	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
14460	return VINF_SUCCESS;
14461	}
14462
14463	/*
14464	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
14465	* The EFER MSR is always up-to-date.
14466	* Update the segment registers and DR7 from the CPU.
14467	*/
14468	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14469	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14470	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
14471	AssertRCReturn(rc, rc);
14472	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
14473
14474	PVM pVM = pVCpu->CTX_SUFF(pVM);
14475	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14476	{
14477	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14478	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
14479	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
14480	if (RT_SUCCESS(rc))
14481	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
14482	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14483	}
14484	else
14485	{
14486	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14487	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
14488	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
14489	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14490	}
14491
14492	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
14493	if (RT_SUCCESS(rc))
14494	{
14495	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14496	AssertRCReturn(rc2, rc2);
14497	return VINF_SUCCESS;
14498	}
14499	return rc;
14500	}
14501
14502
14503	/**
14504	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
14505	* Conditional VM-exit.
14506	*/
14507	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14508	{
14509	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14510	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14511
14512	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14513	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14514	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14515	{
14516	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
14517	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
14518	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14519	{
14520	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14521	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14522	}
14523	}
14524	else
14525	{
14526	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14527	rcStrict1 = VINF_SUCCESS;
14528	return rcStrict1;
14529	}
14530
14531	/*
14532	* Get sufficent state and update the exit history entry.
14533	*/
14534	RTGCPHYS GCPhys;
14535	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14536	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14537	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14538	AssertRCReturn(rc, rc);
14539
14540	VBOXSTRICTRC rcStrict;
14541	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14542	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
14543	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14544	if (!pExitRec)
14545	{
14546	/*
14547	* If we succeed, resume guest execution.
14548	* If we fail in interpreting the instruction because we couldn't get the guest physical address
14549	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
14550	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
14551	* weird case. See @bugref{6043}.
14552	*/
14553	PVM pVM = pVCpu->CTX_SUFF(pVM);
14554	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14555	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
14556	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
14557	if ( rcStrict == VINF_SUCCESS
14558	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
14559	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
14560	{
14561	/* Successfully handled MMIO operation. */
14562	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14563	\| HM_CHANGED_GUEST_APIC_TPR);
14564	rcStrict = VINF_SUCCESS;
14565	}
14566	}
14567	else
14568	{
14569	/*
14570	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14571	*/
14572	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14573	AssertRCReturn(rc2, rc2);
14574
14575	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
14576	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
14577
14578	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14579	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14580
14581	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14582	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14583	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14584	}
14585	return VBOXSTRICTRC_TODO(rcStrict);
14586	}
14587
14588
14589	/**
14590	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
14591	* VM-exit.
14592	*/
14593	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14594	{
14595	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14596	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14597
14598	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14599	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14600	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14601	{
14602	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
14603	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14604	Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
14605	}
14606	else
14607	{
14608	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14609	rcStrict1 = VINF_SUCCESS;
14610	return rcStrict1;
14611	}
14612
14613	RTGCPHYS GCPhys;
14614	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14615	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14616	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14617	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14618	AssertRCReturn(rc, rc);
14619
14620	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
14621	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
14622
14623	RTGCUINT uErrorCode = 0;
14624	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
14625	uErrorCode \|= X86_TRAP_PF_ID;
14626	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
14627	uErrorCode \|= X86_TRAP_PF_RW;
14628	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
14629	uErrorCode \|= X86_TRAP_PF_P;
14630
14631	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
14632
14633
14634	/* Handle the pagefault trap for the nested shadow table. */
14635	PVM pVM = pVCpu->CTX_SUFF(pVM);
14636	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14637
14638	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
14639	pCtx->cs.Sel, pCtx->rip));
14640
14641	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
14642	TRPMResetTrap(pVCpu);
14643
14644	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
14645	if ( rcStrict2 == VINF_SUCCESS
14646	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14647	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14648	{
14649	/* Successfully synced our nested page tables. */
14650	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
14651	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
14652	return VINF_SUCCESS;
14653	}
14654
14655	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14656	return rcStrict2;
14657	}
14658
14659	/** @} */
14660
14661	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14662	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14663	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14664
14665	/**
14666	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
14667	*/
14668	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14669	{
14670	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14671	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
14672
14673	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
14674	AssertRCReturn(rc, rc);
14675
14676	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
14677	{
14678	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
14679	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
14680
14681	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
14682	* provides VM-exit instruction length. If this causes problem later,
14683	* disassemble the instruction like it's done on AMD-V. */
14684	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14685	AssertRCReturn(rc2, rc2);
14686	return rc;
14687	}
14688
14689	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14690	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14691	return rc;
14692	}
14693
14694
14695	/**
14696	* VM-exit exception handler for \#BP (Breakpoint exception).
14697	*/
14698	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14699	{
14700	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14701	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
14702
14703	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14704	AssertRCReturn(rc, rc);
14705
14706	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14707	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14708	if (rc == VINF_EM_RAW_GUEST_TRAP)
14709	{
14710	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14711	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14712	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14713	AssertRCReturn(rc, rc);
14714
14715	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14716	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14717	}
14718
14719	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
14720	return rc;
14721	}
14722
14723
14724	/**
14725	* VM-exit exception handler for \#AC (alignment check exception).
14726	*/
14727	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14728	{
14729	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14730
14731	/*
14732	* Re-inject it. We'll detect any nesting before getting here.
14733	*/
14734	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14735	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14736	AssertRCReturn(rc, rc);
14737	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
14738
14739	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14740	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14741	return VINF_SUCCESS;
14742	}
14743
14744
14745	/**
14746	* VM-exit exception handler for \#DB (Debug exception).
14747	*/
14748	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14749	{
14750	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14751	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
14752
14753	/*
14754	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
14755	* for processing.
14756	*/
14757	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14758
14759	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
14760	uint64_t const uDR6 = X86_DR6_INIT_VAL
14761	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
14762	\| X86_DR6_BD \| X86_DR6_BS));
14763
14764	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14765	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
14766	Log6Func(("rc=%Rrc\n", rc));
14767	if (rc == VINF_EM_RAW_GUEST_TRAP)
14768	{
14769	/*
14770	* The exception was for the guest. Update DR6, DR7.GD and
14771	* IA32_DEBUGCTL.LBR before forwarding it.
14772	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
14773	*/
14774	VMMRZCallRing3Disable(pVCpu);
14775	HM_DISABLE_PREEMPT(pVCpu);
14776
14777	pCtx->dr[6] &= ~X86_DR6_B_MASK;
14778	pCtx->dr[6] \|= uDR6;
14779	if (CPUMIsGuestDebugStateActive(pVCpu))
14780	ASMSetDR6(pCtx->dr[6]);
14781
14782	HM_RESTORE_PREEMPT();
14783	VMMRZCallRing3Enable(pVCpu);
14784
14785	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
14786	AssertRCReturn(rc, rc);
14787
14788	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
14789	pCtx->dr[7] &= ~X86_DR7_GD;
14790
14791	/* Paranoia. */
14792	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
14793	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
14794
14795	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
14796	AssertRCReturn(rc, rc);
14797
14798	/*
14799	* Raise #DB in the guest.
14800	*
14801	* It is important to reflect exactly what the VM-exit gave us (preserving the
14802	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
14803	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
14804	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
14805	*
14806	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
14807	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
14808	*/
14809	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14810	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14811	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14812	AssertRCReturn(rc, rc);
14813	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14814	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14815	return VINF_SUCCESS;
14816	}
14817
14818	/*
14819	* Not a guest trap, must be a hypervisor related debug event then.
14820	* Update DR6 in case someone is interested in it.
14821	*/
14822	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
14823	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
14824	CPUMSetHyperDR6(pVCpu, uDR6);
14825
14826	return rc;
14827	}
14828
14829
14830	/**
14831	* Hacks its way around the lovely mesa driver's backdoor accesses.
14832	*
14833	* @sa hmR0SvmHandleMesaDrvGp
14834	*/
14835	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14836	{
14837	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
14838	RT_NOREF(pCtx);
14839
14840	/* For now we'll just skip the instruction. */
14841	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14842	}
14843
14844
14845	/**
14846	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
14847	* backdoor logging w/o checking what it is running inside.
14848	*
14849	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
14850	* backdoor port and magic numbers loaded in registers.
14851	*
14852	* @returns true if it is, false if it isn't.
14853	* @sa hmR0SvmIsMesaDrvGp
14854	*/
14855	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14856	{
14857	/* 0xed: IN eAX,dx */
14858	uint8_t abInstr[1];
14859	if (pVmxTransient->cbInstr != sizeof(abInstr))
14860	return false;
14861
14862	/* Check that it is #GP(0). */
14863	if (pVmxTransient->uExitIntErrorCode != 0)
14864	return false;
14865
14866	/* Check magic and port. */
14867	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
14868	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
14869	if (pCtx->rax != UINT32_C(0x564d5868))
14870	return false;
14871	if (pCtx->dx != UINT32_C(0x5658))
14872	return false;
14873
14874	/* Flat ring-3 CS. */
14875	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
14876	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
14877	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
14878	if (pCtx->cs.Attr.n.u2Dpl != 3)
14879	return false;
14880	if (pCtx->cs.u64Base != 0)
14881	return false;
14882
14883	/* Check opcode. */
14884	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
14885	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
14886	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
14887	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
14888	if (RT_FAILURE(rc))
14889	return false;
14890	if (abInstr[0] != 0xed)
14891	return false;
14892
14893	return true;
14894	}
14895
14896
14897	/**
14898	* VM-exit exception handler for \#GP (General-protection exception).
14899	*
14900	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
14901	*/
14902	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14903	{
14904	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14905	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
14906
14907	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14908	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14909	if (pVmcsInfo->RealMode.fRealOnV86Active)
14910	{ /* likely */ }
14911	else
14912	{
14913	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14914	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv);
14915	#endif
14916	/* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
14917	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14918	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14919	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14920	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14921	AssertRCReturn(rc, rc);
14922	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
14923	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
14924
14925	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
14926	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
14927	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14928	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14929	else
14930	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
14931	return rc;
14932	}
14933
14934	Assert(CPUMIsGuestInRealModeEx(pCtx));
14935	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
14936
14937	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14938	AssertRCReturn(rc, rc);
14939
14940	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
14941	if (rcStrict == VINF_SUCCESS)
14942	{
14943	if (!CPUMIsGuestInRealModeEx(pCtx))
14944	{
14945	/*
14946	* The guest is no longer in real-mode, check if we can continue executing the
14947	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
14948	*/
14949	pVmcsInfo->RealMode.fRealOnV86Active = false;
14950	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
14951	{
14952	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
14953	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14954	}
14955	else
14956	{
14957	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
14958	rcStrict = VINF_EM_RESCHEDULE;
14959	}
14960	}
14961	else
14962	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14963	}
14964	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14965	{
14966	rcStrict = VINF_SUCCESS;
14967	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14968	}
14969	return VBOXSTRICTRC_VAL(rcStrict);
14970	}
14971
14972
14973	/**
14974	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
14975	* the exception reported in the VMX transient structure back into the VM.
14976	*
14977	* @remarks Requires uExitIntInfo in the VMX transient structure to be
14978	* up-to-date.
14979	*/
14980	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14981	{
14982	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14983	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14984	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14985	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active,
14986	("uVector=%#x u32XcptBitmap=%#X32\n",
14987	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
14988	NOREF(pVmcsInfo);
14989	#endif
14990
14991	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
14992	hmR0VmxCheckExitDueToEventDelivery(). */
14993	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14994	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14995	AssertRCReturn(rc, rc);
14996	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
14997
14998	#ifdef DEBUG_ramshankar
14999	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
15000	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n",
15001	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pCtx->cs.Sel, pCtx->rip));
15002	#endif
15003
15004	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15005	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15006	return VINF_SUCCESS;
15007	}
15008
15009
15010	/**
15011	* VM-exit exception handler for \#PF (Page-fault exception).
15012	*/
15013	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15014	{
15015	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15016	PVM pVM = pVCpu->CTX_SUFF(pVM);
15017	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15018	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15019	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15020	AssertRCReturn(rc, rc);
15021
15022	if (!pVM->hm.s.fNestedPaging)
15023	{ /* likely */ }
15024	else
15025	{
15026	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
15027	Assert(pVCpu->hm.s.fUsingDebugLoop);
15028	#endif
15029	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
15030	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
15031	{
15032	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15033	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
15034	}
15035	else
15036	{
15037	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15038	hmR0VmxSetPendingXcptDF(pVCpu);
15039	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
15040	}
15041	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15042	return rc;
15043	}
15044
15045	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
15046	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
15047	if (pVmxTransient->fVectoringPF)
15048	{
15049	Assert(pVCpu->hm.s.Event.fPending);
15050	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15051	}
15052
15053	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15054	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15055	AssertRCReturn(rc, rc);
15056
15057	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
15058	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
15059
15060	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
15061	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
15062
15063	Log4Func(("#PF: rc=%Rrc\n", rc));
15064	if (rc == VINF_SUCCESS)
15065	{
15066	/*
15067	* This is typically a shadow page table sync or a MMIO instruction. But we may have
15068	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
15069	*/
15070	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15071	TRPMResetTrap(pVCpu);
15072	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
15073	return rc;
15074	}
15075
15076	if (rc == VINF_EM_RAW_GUEST_TRAP)
15077	{
15078	if (!pVmxTransient->fVectoringDoublePF)
15079	{
15080	/* It's a guest page fault and needs to be reflected to the guest. */
15081	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
15082	TRPMResetTrap(pVCpu);
15083	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
15084	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15085	uGstErrorCode, pVmxTransient->uExitQual);
15086	}
15087	else
15088	{
15089	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15090	TRPMResetTrap(pVCpu);
15091	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
15092	hmR0VmxSetPendingXcptDF(pVCpu);
15093	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
15094	}
15095
15096	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15097	return VINF_SUCCESS;
15098	}
15099
15100	TRPMResetTrap(pVCpu);
15101	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
15102	return rc;
15103	}
15104
15105	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15106	/** @name VMX instruction handlers.
15107	* @{
15108	*/
15109	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15110	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VMX instructions VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15111	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15112
15113	/**
15114	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
15115	*/
15116	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15117	{
15118	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15119
15120	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15121	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15122	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15123	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15124	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15125	AssertRCReturn(rc, rc);
15126
15127	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15128
15129	VMXVEXITINFO ExitInfo;
15130	RT_ZERO(ExitInfo);
15131	ExitInfo.uReason = pVmxTransient->uExitReason;
15132	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15133	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15134	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15135	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15136
15137	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
15138	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15139	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15140	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15141	{
15142	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15143	rcStrict = VINF_SUCCESS;
15144	}
15145	return rcStrict;
15146	}
15147
15148
15149	/**
15150	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
15151	*/
15152	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15153	{
15154	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15155
15156	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
15157	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15158	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15159	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15160	AssertRCReturn(rc, rc);
15161
15162	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15163
15164	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
15165	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15166	{
15167	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15168	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15169	}
15170	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15171	return rcStrict;
15172	}
15173
15174
15175	/**
15176	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
15177	*/
15178	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15179	{
15180	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15181
15182	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15183	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15184	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15185	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15186	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15187	AssertRCReturn(rc, rc);
15188
15189	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15190
15191	VMXVEXITINFO ExitInfo;
15192	RT_ZERO(ExitInfo);
15193	ExitInfo.uReason = pVmxTransient->uExitReason;
15194	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15195	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15196	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15197	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15198
15199	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
15200	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15201	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15202	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15203	{
15204	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15205	rcStrict = VINF_SUCCESS;
15206	}
15207	return rcStrict;
15208	}
15209
15210
15211	/**
15212	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
15213	*/
15214	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15215	{
15216	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15217
15218	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15219	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15220	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15221	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15222	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15223	AssertRCReturn(rc, rc);
15224
15225	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15226
15227	VMXVEXITINFO ExitInfo;
15228	RT_ZERO(ExitInfo);
15229	ExitInfo.uReason = pVmxTransient->uExitReason;
15230	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15231	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15232	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15233	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15234
15235	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
15236	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15237	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15238	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15239	{
15240	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15241	rcStrict = VINF_SUCCESS;
15242	}
15243	return rcStrict;
15244	}
15245
15246
15247	/**
15248	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Unconditional VM-exit.
15249	*/
15250	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15251	{
15252	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15253
15254	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15255	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15256	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15257	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15258	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15259	AssertRCReturn(rc, rc);
15260
15261	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15262
15263	VMXVEXITINFO ExitInfo;
15264	RT_ZERO(ExitInfo);
15265	ExitInfo.uReason = pVmxTransient->uExitReason;
15266	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15267	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15268	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15269	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15270	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15271
15272	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
15273	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15274	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15275	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15276	{
15277	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15278	rcStrict = VINF_SUCCESS;
15279	}
15280	return rcStrict;
15281	}
15282
15283
15284	/**
15285	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
15286	*/
15287	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15288	{
15289	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15290
15291	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
15292	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15293	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15294	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15295	AssertRCReturn(rc, rc);
15296
15297	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15298
15299	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
15300	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15301	{
15302	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15303	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15304	}
15305	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15306	return rcStrict;
15307	}
15308
15309
15310	/**
15311	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Unconditional VM-exit.
15312	*/
15313	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15314	{
15315	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15316
15317	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15318	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15319	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15320	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15321	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15322	AssertRCReturn(rc, rc);
15323
15324	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15325
15326	VMXVEXITINFO ExitInfo;
15327	RT_ZERO(ExitInfo);
15328	ExitInfo.uReason = pVmxTransient->uExitReason;
15329	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15330	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15331	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15332	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15333	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15334
15335	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
15336	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15337	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15338	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15339	{
15340	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15341	rcStrict = VINF_SUCCESS;
15342	}
15343	return rcStrict;
15344	}
15345
15346
15347	/**
15348	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
15349	*/
15350	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15351	{
15352	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15353
15354	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15355	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
15356	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15357	AssertRCReturn(rc, rc);
15358
15359	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15360
15361	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
15362	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15363	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
15364	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15365	{
15366	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15367	rcStrict = VINF_SUCCESS;
15368	}
15369	return rcStrict;
15370	}
15371
15372
15373	/**
15374	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
15375	*/
15376	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15377	{
15378	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15379
15380	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15381	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15382	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15383	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15384	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15385	AssertRCReturn(rc, rc);
15386
15387	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15388
15389	VMXVEXITINFO ExitInfo;
15390	RT_ZERO(ExitInfo);
15391	ExitInfo.uReason = pVmxTransient->uExitReason;
15392	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15393	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15394	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15395	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15396
15397	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
15398	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15399	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15400	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15401	{
15402	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15403	rcStrict = VINF_SUCCESS;
15404	}
15405	return rcStrict;
15406	}
15407
15408	/** @} */
15409	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
15410

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 78287

Download in other formats: