HMVMXR0.cpp@ 78230

Last change on this file since 78230 was 78230, checked in by vboxsync, 6 years ago
VMM/HMVMXR0: Nested VMX: bugref:9180 Build fix.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 642.1 KB

Line
1	/* $Id: HMVMXR0.cpp 78230 2019-04-20 05:50:24Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27
28	#include <VBox/vmm/pdmapi.h>
29	#include <VBox/vmm/dbgf.h>
30	#include <VBox/vmm/iem.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#include <VBox/vmm/em.h>
35	#include <VBox/vmm/gim.h>
36	#include <VBox/vmm/apic.h>
37	#ifdef VBOX_WITH_REM
38	# include <VBox/vmm/rem.h>
39	#endif
40	#include "HMInternal.h"
41	#include <VBox/vmm/vm.h>
42	#include <VBox/vmm/hmvmxinline.h>
43	#include "HMVMXR0.h"
44	#include "dtrace/VBoxVMM.h"
45
46	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
47	#ifdef DEBUG_ramshankar
48	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
49	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
50	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
51	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
52	# define HMVMX_ALWAYS_TRAP_PF
53	# define HMVMX_ALWAYS_FLUSH_TLB
54	# define HMVMX_ALWAYS_SWAP_EFER
55	#endif
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	/** Use the function table. */
62	#define HMVMX_USE_FUNCTION_TABLE
63
64	/** Determine which tagged-TLB flush handler to use. */
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70	/** @name HMVMX_READ_XXX
71	* Flags to skip redundant reads of some common VMCS fields that are not part of
72	* the guest-CPU or VCPU state but are needed while handling VM-exits.
73	*/
74	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
75	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
76	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
77	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
78	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
79	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
80	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
81	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
82	/** @} */
83
84	/**
85	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
86	* guest using hardware-assisted VMX.
87	*
88	* This excludes state like GPRs (other than RSP) which are always are
89	* swapped and restored across the world-switch and also registers like EFER,
90	* MSR which cannot be modified by the guest without causing a VM-exit.
91	*/
92	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
93	\| CPUMCTX_EXTRN_RFLAGS \
94	\| CPUMCTX_EXTRN_RSP \
95	\| CPUMCTX_EXTRN_SREG_MASK \
96	\| CPUMCTX_EXTRN_TABLE_MASK \
97	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
98	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
99	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
100	\| CPUMCTX_EXTRN_TSC_AUX \
101	\| CPUMCTX_EXTRN_OTHER_MSRS \
102	\| CPUMCTX_EXTRN_CR0 \
103	\| CPUMCTX_EXTRN_CR3 \
104	\| CPUMCTX_EXTRN_CR4 \
105	\| CPUMCTX_EXTRN_DR7 \
106	\| CPUMCTX_EXTRN_HM_VMX_MASK)
107
108	/**
109	* Exception bitmap mask for real-mode guests (real-on-v86).
110	*
111	* We need to intercept all exceptions manually except:
112	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
113	* due to bugs in Intel CPUs.
114	* - \#PF need not be intercepted even in real-mode if we have nested paging
115	* support.
116	*/
117	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
118	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
119	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
120	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
121	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
122	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
123	\| RT_BIT(X86_XCPT_XF))
124
125	/** Maximum VM-instruction error number. */
126	#define HMVMX_INSTR_ERROR_MAX 28
127
128	/** Profiling macro. */
129	#ifdef HM_PROFILE_EXIT_DISPATCH
130	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
131	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
132	#else
133	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
134	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
135	#endif
136
137	/** Assert that preemption is disabled or covered by thread-context hooks. */
138	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
139	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
140
141	/** Assert that we haven't migrated CPUs when thread-context hooks are not
142	* used. */
143	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
144	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
145	("Illegal migration! Entered on CPU %u Current %u\n", \
146	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
147
148	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
149	* context. */
150	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
151	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
152	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
153
154	/** Helper macro for VM-exit handlers called unexpectedly. */
155	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
156	do { \
157	(a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
158	return VERR_VMX_UNEXPECTED_EXIT; \
159	} while (0)
160
161	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
162	/** Macro that does the necessary privilege checks and intercepted VM-exits for
163	* guests that attempted to execute a VMX instruction. */
164	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
165	do \
166	{ \
167	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
168	if (rcStrictTmp == VINF_SUCCESS) \
169	{ /* likely */ } \
170	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
171	{ \
172	Assert((a_pVCpu)->hm.s.Event.fPending); \
173	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
174	return VINF_SUCCESS; \
175	} \
176	else \
177	{ \
178	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
179	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
180	} \
181	} while (0)
182
183	/** Macro that decodes a memory operand for an instruction VM-exit. */
184	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
185	do \
186	{ \
187	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
188	(a_pGCPtrEffAddr)); \
189	if (rcStrictTmp == VINF_SUCCESS) \
190	{ /* likely */ } \
191	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
192	{ \
193	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
194	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
195	NOREF(uXcptTmp); \
196	return VINF_SUCCESS; \
197	} \
198	else \
199	{ \
200	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
201	return rcStrictTmp; \
202	} \
203	} while (0)
204
205	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
206
207
208	/*********************************************************************************************************************************
209	* Structures and Typedefs *
210	*********************************************************************************************************************************/
211	/**
212	* VMX transient state.
213	*
214	* A state structure for holding miscellaneous information across
215	* VMX non-root operation and restored after the transition.
216	*/
217	typedef struct VMXTRANSIENT
218	{
219	/** The host's rflags/eflags. */
220	RTCCUINTREG fEFlags;
221	#if HC_ARCH_BITS == 32
222	uint32_t u32Alignment0;
223	#endif
224	/** The guest's TPR value used for TPR shadowing. */
225	uint8_t u8GuestTpr;
226	/** Alignment. */
227	uint8_t abAlignment0[7];
228
229	/** The basic VM-exit reason. */
230	uint16_t uExitReason;
231	/** Alignment. */
232	uint16_t u16Alignment0;
233	/** The VM-exit interruption error code. */
234	uint32_t uExitIntErrorCode;
235	/** The VM-exit exit code qualification. */
236	uint64_t uExitQual;
237	/** The Guest-linear address. */
238	uint64_t uGuestLinearAddr;
239
240	/** The VM-exit interruption-information field. */
241	uint32_t uExitIntInfo;
242	/** The VM-exit instruction-length field. */
243	uint32_t cbInstr;
244	/** The VM-exit instruction-information field. */
245	VMXEXITINSTRINFO ExitInstrInfo;
246	/** Whether the VM-entry failed or not. */
247	bool fVMEntryFailed;
248	/** Whether we are currently executing a nested-guest. */
249	bool fIsNestedGuest;
250	/** Alignment. */
251	uint8_t abAlignment1[2];
252
253	/** The VM-entry interruption-information field. */
254	uint32_t uEntryIntInfo;
255	/** The VM-entry exception error code field. */
256	uint32_t uEntryXcptErrorCode;
257	/** The VM-entry instruction length field. */
258	uint32_t cbEntryInstr;
259
260	/** IDT-vectoring information field. */
261	uint32_t uIdtVectoringInfo;
262	/** IDT-vectoring error code. */
263	uint32_t uIdtVectoringErrorCode;
264
265	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
266	uint32_t fVmcsFieldsRead;
267
268	/** Whether the guest debug state was active at the time of VM-exit. */
269	bool fWasGuestDebugStateActive;
270	/** Whether the hyper debug state was active at the time of VM-exit. */
271	bool fWasHyperDebugStateActive;
272	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
273	bool fUpdatedTscOffsettingAndPreemptTimer;
274	/** Whether the VM-exit was caused by a page-fault during delivery of a
275	* contributory exception or a page-fault. */
276	bool fVectoringDoublePF;
277	/** Whether the VM-exit was caused by a page-fault during delivery of an
278	* external interrupt or NMI. */
279	bool fVectoringPF;
280	bool afAlignment0[3];
281
282	/** The VMCS info. object. */
283	PVMXVMCSINFO pVmcsInfo;
284	} VMXTRANSIENT;
285	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
286	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
287	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
288	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
289	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
290	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
291	/** Pointer to VMX transient state. */
292	typedef VMXTRANSIENT *PVMXTRANSIENT;
293
294	/**
295	* Memory operand read or write access.
296	*/
297	typedef enum VMXMEMACCESS
298	{
299	VMXMEMACCESS_READ = 0,
300	VMXMEMACCESS_WRITE = 1
301	} VMXMEMACCESS;
302
303	/**
304	* VMX VM-exit handler.
305	*
306	* @returns Strict VBox status code (i.e. informational status codes too).
307	* @param pVCpu The cross context virtual CPU structure.
308	* @param pVmxTransient The VMX-transient structure.
309	*/
310	#ifndef HMVMX_USE_FUNCTION_TABLE
311	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
312	#else
313	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
314	/** Pointer to VM-exit handler. */
315	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
316	#endif
317
318	/**
319	* VMX VM-exit handler, non-strict status code.
320	*
321	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
322	*
323	* @returns VBox status code, no informational status code returned.
324	* @param pVCpu The cross context virtual CPU structure.
325	* @param pVmxTransient The VMX-transient structure.
326	*
327	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
328	* use of that status code will be replaced with VINF_EM_SOMETHING
329	* later when switching over to IEM.
330	*/
331	#ifndef HMVMX_USE_FUNCTION_TABLE
332	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
333	#else
334	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
335	#endif
336
337
338	/*********************************************************************************************************************************
339	* Internal Functions *
340	*********************************************************************************************************************************/
341	#ifndef HMVMX_USE_FUNCTION_TABLE
342	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
343	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
344	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
345	#else
346	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
347	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
348	#endif
349	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
350	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
351	#endif
352
353	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
354	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
355	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
356	#endif
357
358	/** @name VM-exit handlers.
359	* @{
360	*/
361	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
362	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
363	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
364	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
365	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
366	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
367	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
368	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
369	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
370	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
371	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
372	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
373	static FNVMXEXITHANDLER hmR0VmxExitHlt;
374	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
375	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
376	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
377	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
378	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
379	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
380	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
381	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
382	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
383	static FNVMXEXITHANDLER hmR0VmxExitVmread;
384	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
385	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
386	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
387	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
388	#endif
389	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
390	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
391	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
392	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
393	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
394	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
395	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
396	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
397	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
398	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
399	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
400	static FNVMXEXITHANDLER hmR0VmxExitMwait;
401	static FNVMXEXITHANDLER hmR0VmxExitMtf;
402	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
403	static FNVMXEXITHANDLER hmR0VmxExitPause;
404	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
405	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
406	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
407	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
408	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
409	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
410	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
411	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
412	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
413	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
414	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
415	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
416	/** @} */
417
418	/** @name Helpers for hardware exceptions VM-exit handlers.
419	* @{
420	*/
421	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
422	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
423	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
424	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
425	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
426	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
427	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
428	/** @} */
429
430
431	/*********************************************************************************************************************************
432	* Global Variables *
433	*********************************************************************************************************************************/
434	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
435	static const uint32_t g_aVmcsCacheSegBase[] =
436	{
437	VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
438	VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
439	VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
440	VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
441	VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
442	VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
443	};
444	AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
445	#endif
446	static const uint32_t g_aVmcsSegBase[] =
447	{
448	VMX_VMCS_GUEST_ES_BASE,
449	VMX_VMCS_GUEST_CS_BASE,
450	VMX_VMCS_GUEST_SS_BASE,
451	VMX_VMCS_GUEST_DS_BASE,
452	VMX_VMCS_GUEST_FS_BASE,
453	VMX_VMCS_GUEST_GS_BASE
454	};
455	static const uint32_t g_aVmcsSegSel[] =
456	{
457	VMX_VMCS16_GUEST_ES_SEL,
458	VMX_VMCS16_GUEST_CS_SEL,
459	VMX_VMCS16_GUEST_SS_SEL,
460	VMX_VMCS16_GUEST_DS_SEL,
461	VMX_VMCS16_GUEST_FS_SEL,
462	VMX_VMCS16_GUEST_GS_SEL
463	};
464	static const uint32_t g_aVmcsSegLimit[] =
465	{
466	VMX_VMCS32_GUEST_ES_LIMIT,
467	VMX_VMCS32_GUEST_CS_LIMIT,
468	VMX_VMCS32_GUEST_SS_LIMIT,
469	VMX_VMCS32_GUEST_DS_LIMIT,
470	VMX_VMCS32_GUEST_FS_LIMIT,
471	VMX_VMCS32_GUEST_GS_LIMIT
472	};
473	static const uint32_t g_aVmcsSegAttr[] =
474	{
475	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
476	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
477	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
478	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
479	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
480	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
481	};
482	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
483	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
484	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
485	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
486
487	#ifdef HMVMX_USE_FUNCTION_TABLE
488	/**
489	* VMX_EXIT dispatch table.
490	*/
491	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
492	{
493	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
494	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
495	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
496	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
497	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
498	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
499	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
500	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
501	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
502	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
503	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
504	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
505	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
506	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
507	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
508	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
509	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
510	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
511	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
512	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
513	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
514	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
515	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
516	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
517	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
518	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
519	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
520	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
521	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
522	#else
523	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
524	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
525	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
526	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
527	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
528	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
529	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
530	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
531	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
532	#endif
533	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
534	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
535	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
536	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
537	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
538	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
539	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
540	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
541	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
542	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
543	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
544	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
545	/* 40 UNDEFINED */ hmR0VmxExitPause,
546	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
547	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
548	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
549	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
550	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
551	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitXdtrAccess,
552	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitXdtrAccess,
553	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
554	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
555	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
556	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
557	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
558	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
559	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
560	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
561	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
562	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
563	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
564	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
565	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
566	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
567	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
568	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
569	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
570	};
571	#endif /* HMVMX_USE_FUNCTION_TABLE */
572
573	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
574	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
575	{
576	/* 0 */ "(Not Used)",
577	/* 1 */ "VMCALL executed in VMX root operation.",
578	/* 2 */ "VMCLEAR with invalid physical address.",
579	/* 3 */ "VMCLEAR with VMXON pointer.",
580	/* 4 */ "VMLAUNCH with non-clear VMCS.",
581	/* 5 */ "VMRESUME with non-launched VMCS.",
582	/* 6 */ "VMRESUME after VMXOFF",
583	/* 7 */ "VM-entry with invalid control fields.",
584	/* 8 */ "VM-entry with invalid host state fields.",
585	/* 9 */ "VMPTRLD with invalid physical address.",
586	/* 10 */ "VMPTRLD with VMXON pointer.",
587	/* 11 */ "VMPTRLD with incorrect revision identifier.",
588	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
589	/* 13 */ "VMWRITE to read-only VMCS component.",
590	/* 14 */ "(Not Used)",
591	/* 15 */ "VMXON executed in VMX root operation.",
592	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
593	/* 17 */ "VM-entry with non-launched executing VMCS.",
594	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
595	/* 19 */ "VMCALL with non-clear VMCS.",
596	/* 20 */ "VMCALL with invalid VM-exit control fields.",
597	/* 21 */ "(Not Used)",
598	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
599	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
600	/* 24 */ "VMCALL with invalid SMM-monitor features.",
601	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
602	/* 26 */ "VM-entry with events blocked by MOV SS.",
603	/* 27 */ "(Not Used)",
604	/* 28 */ "Invalid operand to INVEPT/INVVPID."
605	};
606	#endif /* VBOX_STRICT */
607
608
609	/**
610	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
611	*
612	* Any bit set in this mask is owned by the host/hypervisor and would cause a
613	* VM-exit when modified by the guest.
614	*
615	* @returns The static CR0 guest/host mask.
616	* @param pVCpu The cross context virtual CPU structure.
617	*/
618	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PVMCPU pVCpu)
619	{
620	/*
621	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
622	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
623	*/
624	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
625	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
626	* and @bugref{6944}. */
627	PVM pVM = pVCpu->CTX_SUFF(pVM);
628	return ( X86_CR0_PE
629	\| X86_CR0_NE
630	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
631	\| X86_CR0_PG
632	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
633	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
634	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
635	}
636
637
638	/**
639	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
640	*
641	* Any bit set in this mask is owned by the host/hypervisor and would cause a
642	* VM-exit when modified by the guest.
643	*
644	* @returns The static CR4 guest/host mask.
645	* @param pVCpu The cross context virtual CPU structure.
646	*/
647	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PVMCPU pVCpu)
648	{
649	/*
650	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
651	* these bits are reserved on hardware that does not support them. Since the
652	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
653	* these bits and handle it depending on whether we expose them to the guest.
654	*/
655	PVM pVM = pVCpu->CTX_SUFF(pVM);
656	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
657	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
658	return ( X86_CR4_VMXE
659	\| X86_CR4_VME
660	\| X86_CR4_PAE
661	\| X86_CR4_PGE
662	\| X86_CR4_PSE
663	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
664	\| (fPcid ? X86_CR4_PCIDE : 0));
665	}
666
667
668	/**
669	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
670	* area.
671	*
672	* @returns @c true if it's different, @c false otherwise.
673	* @param pVmcsInfo The VMCS info. object.
674	*/
675	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
676	{
677	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
678	&& pVmcsInfo->pvGuestMsrStore);
679	}
680
681
682	/**
683	* Adds one or more exceptions to the exception bitmap and commits it to the current
684	* VMCS.
685	*
686	* @returns VBox status code.
687	* @param pVmxTransient The VMX-transient structure.
688	* @param uXcptMask The exception(s) to add.
689	*/
690	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
691	{
692	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
693	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
694	if ((uXcptBitmap & uXcptMask) != uXcptMask)
695	{
696	uXcptBitmap \|= uXcptMask;
697	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
698	AssertRCReturn(rc, rc);
699	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
700	}
701	return VINF_SUCCESS;
702	}
703
704
705	/**
706	* Adds an exception to the exception bitmap and commits it to the current VMCS.
707	*
708	* @returns VBox status code.
709	* @param pVmxTransient The VMX-transient structure.
710	* @param uXcpt The exception to add.
711	*/
712	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
713	{
714	Assert(uXcpt <= X86_XCPT_LAST);
715	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
716	}
717
718
719	/**
720	* Remove one or more exceptions from the exception bitmap and commits it to the
721	* current VMCS.
722	*
723	* This takes care of not removing the exception intercept if a nested-guest
724	* requires the exception to be intercepted.
725	*
726	* @returns VBox status code.
727	* @param pVCpu The cross context virtual CPU structure.
728	* @param pVmxTransient The VMX-transient structure.
729	* @param uXcptMask The exception(s) to remove.
730	*/
731	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
732	{
733	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
734	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
735	if (u32XcptBitmap & uXcptMask)
736	{
737	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
738	if (!pVmxTransient->fIsNestedGuest)
739	{ /* likely */ }
740	else
741	{
742	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
743	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
744	}
745	#endif
746	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
747	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
748	\| RT_BIT(X86_XCPT_DE)
749	\| RT_BIT(X86_XCPT_NM)
750	\| RT_BIT(X86_XCPT_TS)
751	\| RT_BIT(X86_XCPT_UD)
752	\| RT_BIT(X86_XCPT_NP)
753	\| RT_BIT(X86_XCPT_SS)
754	\| RT_BIT(X86_XCPT_GP)
755	\| RT_BIT(X86_XCPT_PF)
756	\| RT_BIT(X86_XCPT_MF));
757	#elif defined(HMVMX_ALWAYS_TRAP_PF)
758	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
759	#endif
760	if (uXcptMask)
761	{
762	/* Validate we are not removing any essential exception intercepts. */
763	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
764	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
765	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
766
767	/* Remove it from the exception bitmap. */
768	u32XcptBitmap &= ~uXcptMask;
769
770	/* Commit and update the cache if necessary. */
771	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
772	{
773	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
774	AssertRCReturn(rc, rc);
775	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
776	}
777	}
778	}
779	return VINF_SUCCESS;
780	}
781
782
783	/**
784	* Remove an exceptions from the exception bitmap and commits it to the current
785	* VMCS.
786	*
787	* @returns VBox status code.
788	* @param pVCpu The cross context virtual CPU structure.
789	* @param pVmxTransient The VMX-transient structure.
790	* @param uXcpt The exception to remove.
791	*/
792	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
793	{
794	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
795	}
796
797
798	/**
799	* Loads the VMCS specified by the VMCS info. object.
800	*
801	* @returns VBox status code.
802	* @param pVmcsInfo The VMCS info. object.
803	*/
804	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
805	{
806	Assert(pVmcsInfo);
807	Assert(pVmcsInfo->HCPhysVmcs);
808	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
809
810	if (pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_CLEAR)
811	{
812	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
813	if (RT_SUCCESS(rc))
814	{
815	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
816	return VINF_SUCCESS;
817	}
818	return rc;
819	}
820	return VERR_VMX_INVALID_VMCS_LAUNCH_STATE;
821	}
822
823
824	/**
825	* Clears the VMCS specified by the VMCS info. object.
826	*
827	* @returns VBox status code.
828	* @param pVmcsInfo The VMCS info. object.
829	*/
830	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
831	{
832	Assert(pVmcsInfo);
833	Assert(pVmcsInfo->HCPhysVmcs);
834	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
835
836	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
837	if (RT_SUCCESS(rc))
838	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
839	return rc;
840	}
841
842
843	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
844	/**
845	* Switches the current VMCS to the one specified.
846	*
847	* @returns VBox status code.
848	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
849	* @param pVmcsInfoTo The VMCS info. object we are switching to.
850	*
851	* @remarks Called with interrupts disabled.
852	*/
853	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
854	{
855	Assert(pVmcsInfoFrom);
856	Assert(pVmcsInfoTo);
857
858	/*
859	* Clear the VMCS we are switching out if it has not already been cleared.
860	* This will sync any CPU internal data back to the VMCS.
861	*/
862	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
863	{
864	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
865	if (RT_SUCCESS(rc))
866	{ /* likely */ }
867	else
868	return rc;
869	}
870
871	/*
872	* Clear the VMCS we are switching to if it has not already been cleared.
873	* This will initialize the VMCS launch state to "clear" required for loading it.
874	*
875	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
876	*/
877	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
878	{
879	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
880	if (RT_SUCCESS(rc))
881	{ /* likely */ }
882	else
883	return rc;
884	}
885
886	/*
887	* Finally, load the VMCS we are switching to.
888	*/
889	return hmR0VmxLoadVmcs(pVmcsInfoTo);
890	}
891	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
892
893
894	/**
895	* Updates the VM's last error record.
896	*
897	* If there was a VMX instruction error, reads the error data from the VMCS and
898	* updates VCPU's last error record as well.
899	*
900	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
901	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
902	* VERR_VMX_INVALID_VMCS_FIELD.
903	* @param rc The error code.
904	*/
905	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
906	{
907	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
908	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
909	{
910	AssertPtrReturnVoid(pVCpu);
911	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
912	}
913	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
914	}
915
916
917	#ifdef VBOX_STRICT
918	/**
919	* Reads the VM-entry interruption-information field from the VMCS into the VMX
920	* transient structure.
921	*
922	* @returns VBox status code.
923	* @param pVmxTransient The VMX-transient structure.
924	*
925	* @remarks No-long-jump zone!!!
926	*/
927	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
928	{
929	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
930	AssertRCReturn(rc, rc);
931	return VINF_SUCCESS;
932	}
933
934
935	/**
936	* Reads the VM-entry exception error code field from the VMCS into
937	* the VMX transient structure.
938	*
939	* @returns VBox status code.
940	* @param pVmxTransient The VMX-transient structure.
941	*
942	* @remarks No-long-jump zone!!!
943	*/
944	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
945	{
946	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
947	AssertRCReturn(rc, rc);
948	return VINF_SUCCESS;
949	}
950
951
952	/**
953	* Reads the VM-entry exception error code field from the VMCS into
954	* the VMX transient structure.
955	*
956	* @returns VBox status code.
957	* @param pVmxTransient The VMX-transient structure.
958	*
959	* @remarks No-long-jump zone!!!
960	*/
961	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
962	{
963	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
964	AssertRCReturn(rc, rc);
965	return VINF_SUCCESS;
966	}
967	#endif /* VBOX_STRICT */
968
969
970	/**
971	* Reads the VM-exit interruption-information field from the VMCS into the VMX
972	* transient structure.
973	*
974	* @returns VBox status code.
975	* @param pVmxTransient The VMX-transient structure.
976	*/
977	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
978	{
979	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
980	{
981	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
982	AssertRCReturn(rc,rc);
983	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
984	}
985	return VINF_SUCCESS;
986	}
987
988
989	/**
990	* Reads the VM-exit interruption error code from the VMCS into the VMX
991	* transient structure.
992	*
993	* @returns VBox status code.
994	* @param pVmxTransient The VMX-transient structure.
995	*/
996	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
997	{
998	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
999	{
1000	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1001	AssertRCReturn(rc, rc);
1002	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1003	}
1004	return VINF_SUCCESS;
1005	}
1006
1007
1008	/**
1009	* Reads the VM-exit instruction length field from the VMCS into the VMX
1010	* transient structure.
1011	*
1012	* @returns VBox status code.
1013	* @param pVmxTransient The VMX-transient structure.
1014	*/
1015	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1016	{
1017	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1018	{
1019	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1020	AssertRCReturn(rc, rc);
1021	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1022	}
1023	return VINF_SUCCESS;
1024	}
1025
1026
1027	/**
1028	* Reads the VM-exit instruction-information field from the VMCS into
1029	* the VMX transient structure.
1030	*
1031	* @returns VBox status code.
1032	* @param pVmxTransient The VMX-transient structure.
1033	*/
1034	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1035	{
1036	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1037	{
1038	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1039	AssertRCReturn(rc, rc);
1040	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1041	}
1042	return VINF_SUCCESS;
1043	}
1044
1045
1046	/**
1047	* Reads the VM-exit Qualification from the VMCS into the VMX transient structure.
1048	*
1049	* @returns VBox status code.
1050	* @param pVCpu The cross context virtual CPU structure of the
1051	* calling EMT. (Required for the VMCS cache case.)
1052	* @param pVmxTransient The VMX-transient structure.
1053	*/
1054	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1055	{
1056	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1057	{
1058	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1059	AssertRCReturn(rc, rc);
1060	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1061	}
1062	return VINF_SUCCESS;
1063	}
1064
1065
1066	/**
1067	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1068	*
1069	* @returns VBox status code.
1070	* @param pVCpu The cross context virtual CPU structure of the
1071	* calling EMT. (Required for the VMCS cache case.)
1072	* @param pVmxTransient The VMX-transient structure.
1073	*/
1074	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1075	{
1076	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1077	{
1078	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1079	AssertRCReturn(rc, rc);
1080	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1081	}
1082	return VINF_SUCCESS;
1083	}
1084
1085
1086	/**
1087	* Reads the IDT-vectoring information field from the VMCS into the VMX
1088	* transient structure.
1089	*
1090	* @returns VBox status code.
1091	* @param pVmxTransient The VMX-transient structure.
1092	*
1093	* @remarks No-long-jump zone!!!
1094	*/
1095	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1096	{
1097	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1098	{
1099	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1100	AssertRCReturn(rc, rc);
1101	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1102	}
1103	return VINF_SUCCESS;
1104	}
1105
1106
1107	/**
1108	* Reads the IDT-vectoring error code from the VMCS into the VMX
1109	* transient structure.
1110	*
1111	* @returns VBox status code.
1112	* @param pVmxTransient The VMX-transient structure.
1113	*/
1114	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1115	{
1116	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1117	{
1118	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1119	AssertRCReturn(rc, rc);
1120	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1121	}
1122	return VINF_SUCCESS;
1123	}
1124
1125
1126	/**
1127	* Enters VMX root mode operation on the current CPU.
1128	*
1129	* @returns VBox status code.
1130	* @param pVM The cross context VM structure. Can be
1131	* NULL, after a resume.
1132	* @param HCPhysCpuPage Physical address of the VMXON region.
1133	* @param pvCpuPage Pointer to the VMXON region.
1134	*/
1135	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1136	{
1137	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1138	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1139	Assert(pvCpuPage);
1140	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1141
1142	if (pVM)
1143	{
1144	/* Write the VMCS revision identifier to the VMXON region. */
1145	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1146	}
1147
1148	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1149	RTCCUINTREG fEFlags = ASMIntDisableFlags();
1150
1151	/* Enable the VMX bit in CR4 if necessary. */
1152	RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1153
1154	/* Enter VMX root mode. */
1155	int rc = VMXEnable(HCPhysCpuPage);
1156	if (RT_FAILURE(rc))
1157	{
1158	if (!(uOldCr4 & X86_CR4_VMXE))
1159	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
1160
1161	if (pVM)
1162	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1163	}
1164
1165	/* Restore interrupts. */
1166	ASMSetFlags(fEFlags);
1167	return rc;
1168	}
1169
1170
1171	/**
1172	* Exits VMX root mode operation on the current CPU.
1173	*
1174	* @returns VBox status code.
1175	*/
1176	static int hmR0VmxLeaveRootMode(void)
1177	{
1178	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1179
1180	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1181	RTCCUINTREG fEFlags = ASMIntDisableFlags();
1182
1183	/* If we're for some reason not in VMX root mode, then don't leave it. */
1184	RTCCUINTREG uHostCR4 = ASMGetCR4();
1185
1186	int rc;
1187	if (uHostCR4 & X86_CR4_VMXE)
1188	{
1189	/* Exit VMX root mode and clear the VMX bit in CR4. */
1190	VMXDisable();
1191	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
1192	rc = VINF_SUCCESS;
1193	}
1194	else
1195	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1196
1197	/* Restore interrupts. */
1198	ASMSetFlags(fEFlags);
1199	return rc;
1200	}
1201
1202
1203	/**
1204	* Allocates and maps a physically contiguous page. The allocated page is
1205	* zero'd out (used by various VT-x structures).
1206	*
1207	* @returns IPRT status code.
1208	* @param pMemObj Pointer to the ring-0 memory object.
1209	* @param ppVirt Where to store the virtual address of the
1210	* allocation.
1211	* @param pHCPhys Where to store the physical address of the
1212	* allocation.
1213	*/
1214	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1215	{
1216	AssertPtr(pMemObj);
1217	AssertPtr(ppVirt);
1218	AssertPtr(pHCPhys);
1219	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1220	if (RT_FAILURE(rc))
1221	return rc;
1222	ppVirt = RTR0MemObjAddress(pMemObj);
1223	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1224	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1225	return VINF_SUCCESS;
1226	}
1227
1228
1229	/**
1230	* Frees and unmaps an allocated, physical page.
1231	*
1232	* @param pMemObj Pointer to the ring-0 memory object.
1233	* @param ppVirt Where to re-initialize the virtual address of
1234	* allocation as 0.
1235	* @param pHCPhys Where to re-initialize the physical address of the
1236	* allocation as 0.
1237	*/
1238	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1239	{
1240	AssertPtr(pMemObj);
1241	AssertPtr(ppVirt);
1242	AssertPtr(pHCPhys);
1243	/* NULL is valid, accepted and ignored by the free function below. */
1244	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1245	*pMemObj = NIL_RTR0MEMOBJ;
1246	*ppVirt = NULL;
1247	*pHCPhys = NIL_RTHCPHYS;
1248	}
1249
1250
1251	/**
1252	* Initializes a VMCS info. object.
1253	*
1254	* @param pVmcsInfo The VMCS info. object.
1255	*/
1256	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1257	{
1258	RT_ZERO(*pVmcsInfo);
1259
1260	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1261	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1262	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1263	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1264	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1265	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1266	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1267	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1268	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1269	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1270	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1271	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1272	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1273	}
1274
1275
1276	/**
1277	* Frees the VT-x structures for a VMCS info. object.
1278	*
1279	* @param pVM The cross context VM structure.
1280	* @param pVmcsInfo The VMCS info. object.
1281	*/
1282	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1283	{
1284	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1285
1286	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1287	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1288
1289	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1290	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1291	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1292
1293	hmR0VmxInitVmcsInfo(pVmcsInfo);
1294	}
1295
1296
1297	/**
1298	* Allocates the VT-x structures for a VMCS info. object.
1299	*
1300	* @returns VBox status code.
1301	* @param pVCpu The cross context virtual CPU structure.
1302	* @param pVmcsInfo The VMCS info. object.
1303	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1304	*/
1305	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1306	{
1307	PVM pVM = pVCpu->CTX_SUFF(pVM);
1308
1309	/* Allocate the guest VM control structure (VMCS). */
1310	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1311	if (RT_SUCCESS(rc))
1312	{
1313	if (!fIsNstGstVmcs)
1314	{
1315	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1316	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1317	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1318	{
1319	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1320	NULL /* pR3Ptr /, NULL / pRCPtr */);
1321	}
1322	}
1323	else
1324	{
1325	Assert(pVmcsInfo->HCPhysVirtApic == NIL_RTHCPHYS);
1326	Assert(!pVmcsInfo->pbVirtApic);
1327	}
1328
1329	if (RT_SUCCESS(rc))
1330	{
1331	/*
1332	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1333	* transparent accesses of specific MSRs.
1334	*
1335	* If the condition for enabling MSR bitmaps changes here, don't forget to
1336	* update HMIsMsrBitmapActive().
1337	*
1338	* We don't share MSR bitmaps between the guest and nested-guest as we then
1339	* don't need to care about carefully restoring the guest MSR bitmap.
1340	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1341	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1342	*/
1343	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1344	{
1345	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1346	if (RT_SUCCESS(rc))
1347	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1348	}
1349
1350	if (RT_SUCCESS(rc))
1351	{
1352	/*
1353	* Allocate the VM-entry MSR-load area for the guest MSRs.
1354	*
1355	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1356	* the guest and nested-guest.
1357	*/
1358	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1359	&pVmcsInfo->HCPhysGuestMsrLoad);
1360	if (RT_SUCCESS(rc))
1361	{
1362	/*
1363	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1364	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1365	*/
1366	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1367	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1368	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1369
1370	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1371	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1372	&pVmcsInfo->HCPhysHostMsrLoad);
1373	}
1374	}
1375	}
1376	}
1377
1378	return rc;
1379	}
1380
1381
1382	/**
1383	* Free all VT-x structures for the VM.
1384	*
1385	* @returns IPRT status code.
1386	* @param pVM The cross context VM structure.
1387	*/
1388	static void hmR0VmxStructsFree(PVM pVM)
1389	{
1390	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1391	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1392	#endif
1393	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1394
1395	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1396	{
1397	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1398	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1399	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1400	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1401	if (pVM->cpum.ro.GuestFeatures.fVmx)
1402	{
1403	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1404	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1405	}
1406	#endif
1407	}
1408	}
1409
1410
1411	/**
1412	* Allocate all VT-x structures for the VM.
1413	*
1414	* @returns IPRT status code.
1415	* @param pVM The cross context VM structure.
1416	*/
1417	static int hmR0VmxStructsAlloc(PVM pVM)
1418	{
1419	/*
1420	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1421	* The VMCS size cannot be more than 4096 bytes.
1422	*
1423	* See Intel spec. Appendix A.1 "Basic VMX Information".
1424	*/
1425	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1426	if (cbVmcs <= X86_PAGE_4K_SIZE)
1427	{ /* likely */ }
1428	else
1429	{
1430	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1431	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1432	}
1433
1434	/*
1435	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1436	*/
1437	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1438	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1439	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1440	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1441	#endif
1442
1443	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1444	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1445	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1446
1447	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1448	{
1449	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1450	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1451	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1452	}
1453
1454	/*
1455	* Allocate per-VM VT-x structures.
1456	*/
1457	int rc = VINF_SUCCESS;
1458	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1459	/* Allocate crash-dump magic scratch page. */
1460	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1461	if (RT_FAILURE(rc))
1462	{
1463	hmR0VmxStructsFree(pVM);
1464	return rc;
1465	}
1466	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1467	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1468	#endif
1469
1470	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1471	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1472	{
1473	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1474	&pVM->hm.s.vmx.HCPhysApicAccess);
1475	if (RT_FAILURE(rc))
1476	{
1477	hmR0VmxStructsFree(pVM);
1478	return rc;
1479	}
1480	}
1481
1482	/*
1483	* Initialize per-VCPU VT-x structures.
1484	*/
1485	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1486	{
1487	/* Allocate the guest VMCS structures. */
1488	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1489	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
1490	if (RT_SUCCESS(rc))
1491	{
1492	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1493	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
1494	if (pVM->cpum.ro.GuestFeatures.fVmx)
1495	{
1496	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
1497	if (RT_SUCCESS(rc))
1498	{ /* likely */ }
1499	else
1500	break;
1501	}
1502	#endif
1503	}
1504	else
1505	break;
1506	}
1507
1508	if (RT_FAILURE(rc))
1509	{
1510	hmR0VmxStructsFree(pVM);
1511	return rc;
1512	}
1513
1514	return VINF_SUCCESS;
1515	}
1516
1517
1518	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1519	/**
1520	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
1521	*
1522	* @returns @c true if the MSR is intercepted, @c false otherwise.
1523	* @param pvMsrBitmap The MSR bitmap.
1524	* @param offMsr The MSR byte offset.
1525	* @param iBit The bit offset from the byte offset.
1526	*/
1527	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
1528	{
1529	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
1530	Assert(pbMsrBitmap);
1531	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
1532	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
1533	}
1534	#endif
1535
1536
1537	/**
1538	* Sets the permission bits for the specified MSR in the given MSR bitmap.
1539	*
1540	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
1541	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
1542	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
1543	* the read/write access of this MSR.
1544	*
1545	* @param pVCpu The cross context virtual CPU structure.
1546	* @param pVmcsInfo The VMCS info. object.
1547	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1548	* @param idMsr The MSR value.
1549	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
1550	* include both a read -and- a write permission!
1551	*
1552	* @sa HMGetVmxMsrPermission.
1553	*/
1554	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
1555	{
1556	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
1557	Assert(pbMsrBitmap);
1558	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
1559
1560	/*
1561	* MSR-bitmap Layout:
1562	* Byte index MSR range Interpreted as
1563	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1564	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1565	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1566	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1567	*
1568	* A bit corresponding to an MSR within the above range causes a VM-exit
1569	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
1570	* the MSR range, it always cause a VM-exit.
1571	*
1572	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
1573	*/
1574	uint16_t const offBitmapRead = 0;
1575	uint16_t const offBitmapWrite = 0x800;
1576	uint16_t offMsr;
1577	int32_t iBit;
1578	if (idMsr <= UINT32_C(0x00001fff))
1579	{
1580	offMsr = 0;
1581	iBit = idMsr;
1582	}
1583	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1584	{
1585	offMsr = 0x400;
1586	iBit = idMsr - UINT32_C(0xc0000000);
1587	}
1588	else
1589	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
1590
1591	/*
1592	* Set the MSR read permission.
1593	*/
1594	uint16_t const offMsrRead = offBitmapRead + offMsr;
1595	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
1596	if (fMsrpm & VMXMSRPM_ALLOW_RD)
1597	{
1598	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1599	bool const fClear = !fIsNstGstVmcs ? true
1600	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
1601	#else
1602	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1603	bool const fClear = true;
1604	#endif
1605	if (fClear)
1606	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
1607	}
1608	else
1609	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
1610
1611	/*
1612	* Set the MSR write permission.
1613	*/
1614	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
1615	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
1616	if (fMsrpm & VMXMSRPM_ALLOW_WR)
1617	{
1618	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1619	bool const fClear = !fIsNstGstVmcs ? true
1620	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
1621	#else
1622	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1623	bool const fClear = true;
1624	#endif
1625	if (fClear)
1626	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
1627	}
1628	else
1629	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
1630	}
1631
1632
1633	/**
1634	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1635	* area.
1636	*
1637	* @returns VBox status code.
1638	* @param pVCpu The cross context virtual CPU structure.
1639	* @param pVmcsInfo The VMCS info. object.
1640	* @param cMsrs The number of MSRs.
1641	*/
1642	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
1643	{
1644	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1645	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1646	if (RT_UNLIKELY(cMsrs >= cMaxSupportedMsrs))
1647	{
1648	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1649	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1650	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1651	}
1652
1653	/* Commit the MSR counts to the VMCS and update the cache. */
1654	int rc = VINF_SUCCESS;
1655	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
1656	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1657	if (pVmcsInfo->cExitMsrStore != cMsrs)
1658	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1659	if (pVmcsInfo->cExitMsrLoad != cMsrs)
1660	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1661	AssertRCReturn(rc, rc);
1662
1663	pVmcsInfo->cEntryMsrLoad = cMsrs;
1664	pVmcsInfo->cExitMsrStore = cMsrs;
1665	pVmcsInfo->cExitMsrLoad = cMsrs;
1666
1667	return VINF_SUCCESS;
1668	}
1669
1670
1671	/**
1672	* Adds a new (or updates the value of an existing) guest/host MSR
1673	* pair to be swapped during the world-switch as part of the
1674	* auto-load/store MSR area in the VMCS.
1675	*
1676	* @returns VBox status code.
1677	* @param pVCpu The cross context virtual CPU structure.
1678	* @param pVmxTransient The VMX-transient structure.
1679	* @param idMsr The MSR.
1680	* @param uGuestMsrValue Value of the guest MSR.
1681	* @param fSetReadWrite Whether to set the guest read/write access of this
1682	* MSR (thus not causing a VM-exit).
1683	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1684	* necessary.
1685	*/
1686	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
1687	bool fSetReadWrite, bool fUpdateHostMsr)
1688	{
1689	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1690	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1691	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1692	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1693	uint32_t i;
1694
1695	/* Paranoia. */
1696	Assert(pGuestMsrLoad);
1697
1698	/* Check if the MSR already exists in the VM-entry MSR-load area. */
1699	for (i = 0; i < cMsrs; i++)
1700	{
1701	if (pGuestMsrLoad->u32Msr == idMsr)
1702	break;
1703	pGuestMsrLoad++;
1704	}
1705
1706	bool fAdded = false;
1707	if (i == cMsrs)
1708	{
1709	/* The MSR does not exist, bump the MSR coun to make room for the new MSR. */
1710	++cMsrs;
1711	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1712	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
1713
1714	/* Set the guest to read/write this MSR without causing VM-exits. */
1715	if ( fSetReadWrite
1716	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
1717	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
1718
1719	fAdded = true;
1720	}
1721
1722	/* Update the MSR value for the newly added or already existing MSR. */
1723	pGuestMsrLoad->u32Msr = idMsr;
1724	pGuestMsrLoad->u64Value = uGuestMsrValue;
1725
1726	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
1727	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1728	{
1729	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1730	pGuestMsrStore += i;
1731	pGuestMsrStore->u32Msr = idMsr;
1732	pGuestMsrStore->u64Value = 0;
1733	}
1734
1735	/* Update the corresponding slot in the host MSR area. */
1736	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1737	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad && pHostMsr != pVmcsInfo->pvGuestMsrStore);
1738	pHostMsr += i;
1739	pHostMsr->u32Msr = idMsr;
1740
1741	/*
1742	* Only if the caller requests to update the host MSR value AND we've newly added the
1743	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
1744	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
1745	*
1746	* We do this for performance reasons since reading MSRs may be quite expensive.
1747	*/
1748	if ( fAdded
1749	&& fUpdateHostMsr)
1750	{
1751	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1752	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1753	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1754	}
1755	return VINF_SUCCESS;
1756	}
1757
1758
1759	/**
1760	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1761	* auto-load/store MSR area in the VMCS.
1762	*
1763	* @returns VBox status code.
1764	* @param pVCpu The cross context virtual CPU structure.
1765	* @param pVmxTransient The VMX-transient structure.
1766	* @param idMsr The MSR.
1767	*/
1768	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
1769	{
1770	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1771	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1772	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1773	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1774
1775	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
1776	for (uint32_t i = 0; i < cMsrs; i++)
1777	{
1778	/* Find the MSR. */
1779	if (pGuestMsrLoad->u32Msr == idMsr)
1780	{
1781	/* If it's the last MSR, simply reduce the count. */
1782	if (i == cMsrs - 1)
1783	{
1784	--cMsrs;
1785	break;
1786	}
1787
1788	/* Remove it by copying the last MSR in place of it, and reducing the count. */
1789	PVMXAUTOMSR pLastGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1790	pLastGuestMsrLoad += cMsrs - 1;
1791	pGuestMsrLoad->u32Msr = pLastGuestMsrLoad->u32Msr;
1792	pGuestMsrLoad->u64Value = pLastGuestMsrLoad->u64Value;
1793
1794	/* Remove it from the VM-exit MSR-store area if we are using a different page. */
1795	if (fSeparateExitMsrStorePage)
1796	{
1797	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1798	PVMXAUTOMSR pLastGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1799	pGuestMsrStore += i;
1800	pLastGuestMsrStore += cMsrs - 1;
1801	Assert(pGuestMsrStore->u32Msr == idMsr);
1802	pGuestMsrStore->u32Msr = pLastGuestMsrStore->u32Msr;
1803	pGuestMsrStore->u64Value = pLastGuestMsrStore->u64Value;
1804	}
1805
1806	/* Remove it from the VM-exit MSR-load area. */
1807	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1808	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1809	pHostMsr += i;
1810	pLastHostMsr += cMsrs - 1;
1811	Assert(pHostMsr->u32Msr == idMsr);
1812	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1813	pHostMsr->u64Value = pLastHostMsr->u64Value;
1814	--cMsrs;
1815	break;
1816	}
1817	pGuestMsrLoad++;
1818	}
1819
1820	/* Update the VMCS if the count changed (meaning the MSR was found). */
1821	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
1822	{
1823	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1824	AssertRCReturn(rc, rc);
1825
1826	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1827	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1828	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
1829
1830	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
1831	return VINF_SUCCESS;
1832	}
1833
1834	return VERR_NOT_FOUND;
1835	}
1836
1837
1838	/**
1839	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
1840	*
1841	* @returns @c true if found, @c false otherwise.
1842	* @param pVmcsInfo The VMCS info. object.
1843	* @param idMsr The MSR to find.
1844	*/
1845	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
1846	{
1847	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1848	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
1849	for (uint32_t i = 0; i < cMsrs; i++)
1850	{
1851	if (pGuestMsrLoad->u32Msr == idMsr)
1852	return true;
1853	pGuestMsrLoad++;
1854	}
1855	return false;
1856	}
1857
1858
1859	/**
1860	* Updates the value of all host MSRs in the VM-exit MSR-load area.
1861	*
1862	* @param pVCpu The cross context virtual CPU structure.
1863	* @param pVmcsInfo The VMCS info. object.
1864	*
1865	* @remarks No-long-jump zone!!!
1866	*/
1867	static void hmR0VmxUpdateAutoLoadHostMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
1868	{
1869	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1870	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
1871
1872	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1873	Assert(pHostMsrLoad);
1874
1875	for (uint32_t i = 0; i < cMsrs; i++, pHostMsrLoad++)
1876	{
1877	/*
1878	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1879	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1880	*/
1881	if (pHostMsrLoad->u32Msr == MSR_K6_EFER)
1882	pHostMsrLoad->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
1883	else
1884	pHostMsrLoad->u64Value = ASMRdMsr(pHostMsrLoad->u32Msr);
1885	}
1886	}
1887
1888
1889	/**
1890	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1891	* perform lazy restoration of the host MSRs while leaving VT-x.
1892	*
1893	* @param pVCpu The cross context virtual CPU structure.
1894	*
1895	* @remarks No-long-jump zone!!!
1896	*/
1897	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1898	{
1899	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1900
1901	/*
1902	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
1903	*/
1904	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1905	{
1906	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1907	#if HC_ARCH_BITS == 64
1908	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1909	{
1910	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
1911	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
1912	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
1913	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1914	}
1915	#endif
1916	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1917	}
1918	}
1919
1920
1921	/**
1922	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1923	* lazily while leaving VT-x.
1924	*
1925	* @returns true if it does, false otherwise.
1926	* @param pVCpu The cross context virtual CPU structure.
1927	* @param idMsr The MSR to check.
1928	*/
1929	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t idMsr)
1930	{
1931	NOREF(pVCpu);
1932	#if HC_ARCH_BITS == 64
1933	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1934	{
1935	switch (idMsr)
1936	{
1937	case MSR_K8_LSTAR:
1938	case MSR_K6_STAR:
1939	case MSR_K8_SF_MASK:
1940	case MSR_K8_KERNEL_GS_BASE:
1941	return true;
1942	}
1943	}
1944	#else
1945	RT_NOREF(pVCpu, idMsr);
1946	#endif
1947	return false;
1948	}
1949
1950
1951	/**
1952	* Loads a set of guests MSRs to allow read/passthru to the guest.
1953	*
1954	* The name of this function is slightly confusing. This function does NOT
1955	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1956	* common prefix for functions dealing with "lazy restoration" of the shared
1957	* MSRs.
1958	*
1959	* @param pVCpu The cross context virtual CPU structure.
1960	*
1961	* @remarks No-long-jump zone!!!
1962	*/
1963	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1964	{
1965	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1966	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1967
1968	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1969	#if HC_ARCH_BITS == 64
1970	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1971	{
1972	/*
1973	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1974	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1975	* we can skip a few MSR writes.
1976	*
1977	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1978	* guest MSR values in the guest-CPU context might be different to what's currently
1979	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1980	* CPU, see @bugref{8728}.
1981	*/
1982	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1983	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1984	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
1985	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
1986	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
1987	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
1988	{
1989	#ifdef VBOX_STRICT
1990	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1991	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1992	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1993	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1994	#endif
1995	}
1996	else
1997	{
1998	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1999	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2000	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2001	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2002	}
2003	}
2004	#endif
2005	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2006	}
2007
2008
2009	/**
2010	* Performs lazy restoration of the set of host MSRs if they were previously
2011	* loaded with guest MSR values.
2012	*
2013	* @param pVCpu The cross context virtual CPU structure.
2014	*
2015	* @remarks No-long-jump zone!!!
2016	* @remarks The guest MSRs should have been saved back into the guest-CPU
2017	* context by hmR0VmxImportGuestState()!!!
2018	*/
2019	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2020	{
2021	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2022	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2023
2024	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2025	{
2026	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2027	#if HC_ARCH_BITS == 64
2028	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2029	{
2030	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2031	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2032	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2033	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2034	}
2035	#endif
2036	}
2037	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2038	}
2039
2040
2041	/**
2042	* Verifies that our cached values of the VMCS fields are all consistent with
2043	* what's actually present in the VMCS.
2044	*
2045	* @returns VBox status code.
2046	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2047	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2048	* VMCS content. HMCPU error-field is
2049	* updated, see VMX_VCI_XXX.
2050	* @param pVCpu The cross context virtual CPU structure.
2051	* @param pVmcsInfo The VMCS info. object.
2052	*/
2053	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2054	{
2055	uint32_t u32Val;
2056	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2057	AssertRCReturn(rc, rc);
2058	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2059	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32EntryCtls, u32Val),
2060	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2061	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2062
2063	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2064	AssertRCReturn(rc, rc);
2065	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2066	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ExitCtls, u32Val),
2067	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2068	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2069
2070	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2071	AssertRCReturn(rc, rc);
2072	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2073	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32PinCtls, u32Val),
2074	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2075	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2076
2077	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2078	AssertRCReturn(rc, rc);
2079	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2080	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls, u32Val),
2081	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2082	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2083
2084	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2085	{
2086	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2087	AssertRCReturn(rc, rc);
2088	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2089	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls2, u32Val),
2090	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2091	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2092	}
2093
2094	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2095	AssertRCReturn(rc, rc);
2096	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2097	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32XcptBitmap, u32Val),
2098	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2099	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2100
2101	uint64_t u64Val;
2102	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2103	AssertRCReturn(rc, rc);
2104	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2105	("Cache=%#RX64 VMCS=%#RX64\n", pVmcsInfo->u64TscOffset, u64Val),
2106	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2107	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2108
2109	return VINF_SUCCESS;
2110	}
2111
2112
2113	#ifdef VBOX_STRICT
2114	/**
2115	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2116	*
2117	* @param pVCpu The cross context virtual CPU structure.
2118	* @param pVmcsInfo The VMCS info. object.
2119	*/
2120	static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2121	{
2122	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2123
2124	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2125	{
2126	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2127	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2128	uint64_t uVmcsEferMsrVmcs;
2129	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2130	AssertRC(rc);
2131
2132	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2133	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2134	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2135	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2136	}
2137	}
2138
2139
2140	/**
2141	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2142	* VMCS are correct.
2143	*
2144	* @param pVCpu The cross context virtual CPU structure.
2145	* @param pVmcsInfo The VMCS info. object.
2146	*/
2147	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2148	{
2149	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2150
2151	/* Verify MSR counts in the VMCS are what we think it should be. */
2152	uint32_t cMsrs;
2153	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs);
2154	AssertRC(rc);
2155	Assert(cMsrs == pVmcsInfo->cEntryMsrLoad);
2156
2157	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs);
2158	AssertRC(rc);
2159	Assert(cMsrs == pVmcsInfo->cExitMsrStore);
2160
2161	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs);
2162	AssertRC(rc);
2163	Assert(cMsrs == pVmcsInfo->cExitMsrLoad);
2164
2165	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2166	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2167
2168	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2169	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2170	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2171	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2172	for (uint32_t i = 0; i < cMsrs; i++)
2173	{
2174	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2175	if (fSeparateExitMsrStorePage)
2176	{
2177	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2178	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2179	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2180	}
2181
2182	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2183	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2184	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2185
2186	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2187	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2188	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2189	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2190
2191	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2192	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2193	{
2194	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2195	if (pGuestMsrLoad->u32Msr == MSR_K6_EFER)
2196	{
2197	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2198	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2199	}
2200	else
2201	{
2202	AssertMsgReturnVoid((fMsrpm & (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR))
2203	== (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR),
2204	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2205	}
2206	}
2207
2208	/* Move to the next MSR. */
2209	pHostMsrLoad++;
2210	pGuestMsrLoad++;
2211	pGuestMsrStore++;
2212	}
2213	}
2214	#endif /* VBOX_STRICT */
2215
2216
2217	/**
2218	* Flushes the TLB using EPT.
2219	*
2220	* @returns VBox status code.
2221	* @param pVCpu The cross context virtual CPU structure of the calling
2222	* EMT. Can be NULL depending on @a enmTlbFlush.
2223	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2224	* enmTlbFlush.
2225	* @param enmTlbFlush Type of flush.
2226	*
2227	* @remarks Caller is responsible for making sure this function is called only
2228	* when NestedPaging is supported and providing @a enmTlbFlush that is
2229	* supported by the CPU.
2230	* @remarks Can be called with interrupts disabled.
2231	*/
2232	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2233	{
2234	uint64_t au64Descriptor[2];
2235	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2236	au64Descriptor[0] = 0;
2237	else
2238	{
2239	Assert(pVCpu);
2240	Assert(pVmcsInfo);
2241	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2242	}
2243	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2244
2245	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2246	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2247
2248	if ( RT_SUCCESS(rc)
2249	&& pVCpu)
2250	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2251	}
2252
2253
2254	/**
2255	* Flushes the TLB using VPID.
2256	*
2257	* @returns VBox status code.
2258	* @param pVCpu The cross context virtual CPU structure of the calling
2259	* EMT. Can be NULL depending on @a enmTlbFlush.
2260	* @param enmTlbFlush Type of flush.
2261	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2262	* on @a enmTlbFlush).
2263	*
2264	* @remarks Can be called with interrupts disabled.
2265	*/
2266	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2267	{
2268	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2269
2270	uint64_t au64Descriptor[2];
2271	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2272	{
2273	au64Descriptor[0] = 0;
2274	au64Descriptor[1] = 0;
2275	}
2276	else
2277	{
2278	AssertPtr(pVCpu);
2279	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2280	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2281	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2282	au64Descriptor[1] = GCPtr;
2283	}
2284
2285	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2286	AssertMsg(rc == VINF_SUCCESS,
2287	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2288
2289	if ( RT_SUCCESS(rc)
2290	&& pVCpu)
2291	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2292	NOREF(rc);
2293	}
2294
2295
2296	/**
2297	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2298	* otherwise there is nothing really to invalidate.
2299	*
2300	* @returns VBox status code.
2301	* @param pVCpu The cross context virtual CPU structure.
2302	* @param GCVirt Guest virtual address of the page to invalidate.
2303	*/
2304	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2305	{
2306	AssertPtr(pVCpu);
2307	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2308
2309	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2310	{
2311	/*
2312	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2313	* the EPT case. See @bugref{6043} and @bugref{6177}.
2314	*
2315	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2316	* as this function maybe called in a loop with individual addresses.
2317	*/
2318	PVM pVM = pVCpu->CTX_SUFF(pVM);
2319	if (pVM->hm.s.vmx.fVpid)
2320	{
2321	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2322
2323	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2324	/*
2325	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2326	* where executing INVVPID outside 64-bit mode does not flush translations of
2327	* 64-bit linear addresses, see @bugref{6208#c72}.
2328	*/
2329	if (RT_HI_U32(GCVirt))
2330	fVpidFlush = false;
2331	#endif
2332
2333	if (fVpidFlush)
2334	{
2335	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2336	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2337	}
2338	else
2339	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2340	}
2341	else if (pVM->hm.s.fNestedPaging)
2342	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2343	}
2344
2345	return VINF_SUCCESS;
2346	}
2347
2348
2349	/**
2350	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2351	* case where neither EPT nor VPID is supported by the CPU.
2352	*
2353	* @param pHostCpu The HM physical-CPU structure.
2354	* @param pVCpu The cross context virtual CPU structure.
2355	*
2356	* @remarks Called with interrupts disabled.
2357	*/
2358	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2359	{
2360	AssertPtr(pVCpu);
2361	AssertPtr(pHostCpu);
2362
2363	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2364
2365	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2366	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2367	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2368	pVCpu->hm.s.fForceTLBFlush = false;
2369	return;
2370	}
2371
2372
2373	/**
2374	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2375	*
2376	* @param pHostCpu The HM physical-CPU structure.
2377	* @param pVCpu The cross context virtual CPU structure.
2378	* @param pVmcsInfo The VMCS info. object.
2379	*
2380	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2381	* nomenclature. The reason is, to avoid confusion in compare statements
2382	* since the host-CPU copies are named "ASID".
2383	*
2384	* @remarks Called with interrupts disabled.
2385	*/
2386	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2387	{
2388	#ifdef VBOX_WITH_STATISTICS
2389	bool fTlbFlushed = false;
2390	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2391	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2392	if (!fTlbFlushed) \
2393	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2394	} while (0)
2395	#else
2396	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2397	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2398	#endif
2399
2400	AssertPtr(pVCpu);
2401	AssertPtr(pHostCpu);
2402	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2403
2404	PVM pVM = pVCpu->CTX_SUFF(pVM);
2405	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2406	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2407	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2408
2409	/*
2410	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2411	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2412	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2413	* cannot reuse the current ASID anymore.
2414	*/
2415	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2416	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2417	{
2418	++pHostCpu->uCurrentAsid;
2419	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2420	{
2421	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2422	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2423	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2424	}
2425
2426	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2427	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2428	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2429
2430	/*
2431	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2432	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2433	*/
2434	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2435	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2436	HMVMX_SET_TAGGED_TLB_FLUSHED();
2437	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2438	}
2439	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2440	{
2441	/*
2442	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2443	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2444	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2445	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2446	* mappings, see @bugref{6568}.
2447	*
2448	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2449	*/
2450	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2451	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2452	HMVMX_SET_TAGGED_TLB_FLUSHED();
2453	}
2454
2455	pVCpu->hm.s.fForceTLBFlush = false;
2456	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2457
2458	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
2459	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
2460	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2461	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2462	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2463	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2464	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2465	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2466	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2467
2468	/* Update VMCS with the VPID. */
2469	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2470	AssertRC(rc);
2471
2472	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2473	}
2474
2475
2476	/**
2477	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2478	*
2479	* @param pHostCpu The HM physical-CPU structure.
2480	* @param pVCpu The cross context virtual CPU structure.
2481	* @param pVmcsInfo The VMCS info. object.
2482	*
2483	* @remarks Called with interrupts disabled.
2484	*/
2485	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2486	{
2487	AssertPtr(pVCpu);
2488	AssertPtr(pHostCpu);
2489	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2490	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2491	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2492
2493	/*
2494	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2495	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2496	*/
2497	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2498	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2499	{
2500	pVCpu->hm.s.fForceTLBFlush = true;
2501	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2502	}
2503
2504	/* Check for explicit TLB flushes. */
2505	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2506	{
2507	pVCpu->hm.s.fForceTLBFlush = true;
2508	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2509	}
2510
2511	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2512	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2513
2514	if (pVCpu->hm.s.fForceTLBFlush)
2515	{
2516	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2517	pVCpu->hm.s.fForceTLBFlush = false;
2518	}
2519	}
2520
2521
2522	/**
2523	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2524	*
2525	* @param pHostCpu The HM physical-CPU structure.
2526	* @param pVCpu The cross context virtual CPU structure.
2527	*
2528	* @remarks Called with interrupts disabled.
2529	*/
2530	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2531	{
2532	AssertPtr(pVCpu);
2533	AssertPtr(pHostCpu);
2534	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2535	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2536	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2537
2538	/*
2539	* Force a TLB flush for the first world switch if the current CPU differs from the one we
2540	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2541	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2542	* cannot reuse the current ASID anymore.
2543	*/
2544	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2545	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2546	{
2547	pVCpu->hm.s.fForceTLBFlush = true;
2548	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2549	}
2550
2551	/* Check for explicit TLB flushes. */
2552	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2553	{
2554	/*
2555	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2556	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2557	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
2558	* include fExplicitFlush's too) - an obscure corner case.
2559	*/
2560	pVCpu->hm.s.fForceTLBFlush = true;
2561	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2562	}
2563
2564	PVM pVM = pVCpu->CTX_SUFF(pVM);
2565	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2566	if (pVCpu->hm.s.fForceTLBFlush)
2567	{
2568	++pHostCpu->uCurrentAsid;
2569	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2570	{
2571	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2572	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2573	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2574	}
2575
2576	pVCpu->hm.s.fForceTLBFlush = false;
2577	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2578	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2579	if (pHostCpu->fFlushAsidBeforeUse)
2580	{
2581	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2582	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2583	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2584	{
2585	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2586	pHostCpu->fFlushAsidBeforeUse = false;
2587	}
2588	else
2589	{
2590	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2591	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2592	}
2593	}
2594	}
2595
2596	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2597	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2598	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2599	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2600	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2601	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2602	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2603
2604	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2605	AssertRC(rc);
2606	}
2607
2608
2609	/**
2610	* Flushes the guest TLB entry based on CPU capabilities.
2611	*
2612	* @param pHostCpu The HM physical-CPU structure.
2613	* @param pVCpu The cross context virtual CPU structure.
2614	* @param pVmcsInfo The VMCS info. object.
2615	*
2616	* @remarks Called with interrupts disabled.
2617	*/
2618	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2619	{
2620	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2621	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2622	#endif
2623	PVM pVM = pVCpu->CTX_SUFF(pVM);
2624	switch (pVM->hm.s.vmx.enmTlbFlushType)
2625	{
2626	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
2627	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
2628	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
2629	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
2630	default:
2631	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2632	break;
2633	}
2634	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2635	}
2636
2637
2638	/**
2639	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2640	* TLB entries from the host TLB before VM-entry.
2641	*
2642	* @returns VBox status code.
2643	* @param pVM The cross context VM structure.
2644	*/
2645	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2646	{
2647	/*
2648	* Determine optimal flush type for nested paging.
2649	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
2650	* unrestricted guest execution (see hmR3InitFinalizeR0()).
2651	*/
2652	if (pVM->hm.s.fNestedPaging)
2653	{
2654	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2655	{
2656	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2657	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2658	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2659	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2660	else
2661	{
2662	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2663	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2664	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2665	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2666	}
2667
2668	/* Make sure the write-back cacheable memory type for EPT is supported. */
2669	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2670	{
2671	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2672	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2673	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2674	}
2675
2676	/* EPT requires a page-walk length of 4. */
2677	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2678	{
2679	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2680	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2681	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2682	}
2683	}
2684	else
2685	{
2686	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2687	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2688	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2689	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2690	}
2691	}
2692
2693	/*
2694	* Determine optimal flush type for VPID.
2695	*/
2696	if (pVM->hm.s.vmx.fVpid)
2697	{
2698	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2699	{
2700	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2701	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2702	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2703	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2704	else
2705	{
2706	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2707	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2708	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2709	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2710	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2711	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2712	pVM->hm.s.vmx.fVpid = false;
2713	}
2714	}
2715	else
2716	{
2717	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2718	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2719	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2720	pVM->hm.s.vmx.fVpid = false;
2721	}
2722	}
2723
2724	/*
2725	* Setup the handler for flushing tagged-TLBs.
2726	*/
2727	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2728	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2729	else if (pVM->hm.s.fNestedPaging)
2730	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2731	else if (pVM->hm.s.vmx.fVpid)
2732	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2733	else
2734	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2735	return VINF_SUCCESS;
2736	}
2737
2738
2739	/**
2740	* Sets up the virtual-APIC page address for the VMCS.
2741	*
2742	* @returns VBox status code.
2743	* @param pVCpu The cross context virtual CPU structure.
2744	* @param pVmcsInfo The VMCS info. object.
2745	*/
2746	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2747	{
2748	NOREF(pVCpu);
2749	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
2750	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
2751	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2752	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
2753	}
2754
2755
2756	/**
2757	* Sets up the MSR-bitmap address for the VMCS.
2758	*
2759	* @returns VBox status code.
2760	* @param pVCpu The cross context virtual CPU structure.
2761	* @param pVmcsInfo The VMCS info. object.
2762	*/
2763	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2764	{
2765	NOREF(pVCpu);
2766	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
2767	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
2768	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2769	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
2770	}
2771
2772
2773	/**
2774	* Sets up the APIC-access page address for the VMCS.
2775	*
2776	* @returns VBox status code.
2777	* @param pVCpu The cross context virtual CPU structure.
2778	*/
2779	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
2780	{
2781	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
2782	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
2783	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2784	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
2785	}
2786
2787
2788	/**
2789	* Sets up the VMCS link pointer for the VMCS.
2790	*
2791	* @returns VBox status code.
2792	* @param pVCpu The cross context virtual CPU structure.
2793	* @param pVmcsInfo The VMCS info. object.
2794	*/
2795	DECLINLINE(int) hmR0VmxSetupVmcsLinkPtr(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2796	{
2797	NOREF(pVCpu);
2798	uint64_t const u64VmcsLinkPtr = pVmcsInfo->u64VmcsLinkPtr;
2799	Assert(u64VmcsLinkPtr == UINT64_C(0xffffffffffffffff)); /* Bits 63:0 MB1. */
2800	return VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, u64VmcsLinkPtr);
2801	}
2802
2803
2804	/**
2805	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
2806	* in the VMCS.
2807	*
2808	* @returns VBox status code.
2809	* @param pVCpu The cross context virtual CPU structure.
2810	* @param pVmcsInfo The VMCS info. object.
2811	*/
2812	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2813	{
2814	NOREF(pVCpu);
2815	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
2816	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
2817	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2818
2819	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
2820	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
2821	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
2822
2823	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
2824	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
2825	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2826
2827	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
2828	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
2829	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
2830	AssertRCReturn(rc, rc);
2831	return VINF_SUCCESS;
2832	}
2833
2834
2835	/**
2836	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
2837	*
2838	* @param pVCpu The cross context virtual CPU structure.
2839	* @param pVmcsInfo The VMCS info. object.
2840	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2841	*/
2842	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2843	{
2844	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
2845
2846	/*
2847	* The guest can access the following MSRs (read, write) without causing
2848	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
2849	*/
2850	PVM pVM = pVCpu->CTX_SUFF(pVM);
2851	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2852	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2853	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2854	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2855	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2856
2857	#ifdef VBOX_STRICT
2858	/** @todo NSTVMX: Remove this later. */
2859	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_IA32_SYSENTER_CS);
2860	Assert((fMsrpm & (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR)) == (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR));
2861
2862	fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K8_GS_BASE);
2863	Assert((fMsrpm & (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR)) == (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR));
2864	#endif
2865
2866	/*
2867	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
2868	* associated with then. We never need to intercept access (writes need to be
2869	* executed without causing a VM-exit, reads will #GP fault anyway).
2870	*
2871	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
2872	* read/write them. We swap the the guest/host MSR value using the
2873	* auto-load/store MSR area.
2874	*/
2875	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2876	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2877	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
2878	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2879	if (pVM->cpum.ro.GuestFeatures.fIbrs)
2880	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2881
2882	/*
2883	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
2884	*/
2885
2886	#if HC_ARCH_BITS == 64
2887	/*
2888	* Allow full read/write access for the following MSRs (mandatory for VT-x)
2889	* required for 64-bit guests.
2890	*/
2891	if (pVM->hm.s.fAllow64BitGuests)
2892	{
2893	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2894	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K6_STAR, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2895	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2896	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2897
2898	# ifdef VBOX_STRICT
2899	fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K8_GS_BASE);
2900	Assert((fMsrpm & (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR)) == (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR));
2901	# endif
2902	}
2903	#endif
2904	}
2905
2906
2907	/**
2908	* Sets up pin-based VM-execution controls in the VMCS.
2909	*
2910	* @returns VBox status code.
2911	* @param pVCpu The cross context virtual CPU structure.
2912	* @param pVmcsInfo The VMCS info. object.
2913	*/
2914	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2915	{
2916	PVM pVM = pVCpu->CTX_SUFF(pVM);
2917	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
2918	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2919
2920	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2921	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2922
2923	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2924	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2925
2926	/* Enable the VMX-preemption timer. */
2927	if (pVM->hm.s.vmx.fUsePreemptTimer)
2928	{
2929	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2930	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
2931	}
2932
2933	#if 0
2934	/* Enable posted-interrupt processing. */
2935	if (pVM->hm.s.fPostedIntrs)
2936	{
2937	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2938	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2939	fVal \|= VMX_PIN_CTL_POSTED_INT;
2940	}
2941	#endif
2942
2943	if ((fVal & fZap) != fVal)
2944	{
2945	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2946	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
2947	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2948	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2949	}
2950
2951	/* Commit it to the VMCS and update our cache. */
2952	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2953	AssertRCReturn(rc, rc);
2954	pVmcsInfo->u32PinCtls = fVal;
2955
2956	return VINF_SUCCESS;
2957	}
2958
2959
2960	/**
2961	* Sets up secondary processor-based VM-execution controls in the VMCS.
2962	*
2963	* @returns VBox status code.
2964	* @param pVCpu The cross context virtual CPU structure.
2965	* @param pVmcsInfo The VMCS info. object.
2966	*/
2967	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2968	{
2969	PVM pVM = pVCpu->CTX_SUFF(pVM);
2970	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
2971	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2972
2973	/* WBINVD causes a VM-exit. */
2974	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2975	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
2976
2977	/* Enable EPT (aka nested-paging). */
2978	if (pVM->hm.s.fNestedPaging)
2979	fVal \|= VMX_PROC_CTLS2_EPT;
2980
2981	/* Enable the INVPCID instruction if supported by the hardware and we expose
2982	it to the guest. Without this, guest executing INVPCID would cause a #UD. */
2983	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2984	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
2985	fVal \|= VMX_PROC_CTLS2_INVPCID;
2986
2987	/* Enable VPID. */
2988	if (pVM->hm.s.vmx.fVpid)
2989	fVal \|= VMX_PROC_CTLS2_VPID;
2990
2991	/* Enable unrestricted guest execution. */
2992	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2993	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
2994
2995	#if 0
2996	if (pVM->hm.s.fVirtApicRegs)
2997	{
2998	/* Enable APIC-register virtualization. */
2999	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3000	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3001
3002	/* Enable virtual-interrupt delivery. */
3003	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3004	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3005	}
3006	#endif
3007
3008	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
3009	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3010	* done dynamically. */
3011	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3012	{
3013	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3014	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3015	AssertRCReturn(rc, rc);
3016	}
3017
3018	/* Enable the RDTSCP instruction if supported by the hardware and we expose
3019	it to the guest. Without this, guest executing RDTSCP would cause a #UD. */
3020	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
3021	&& pVM->cpum.ro.GuestFeatures.fRdTscP)
3022	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3023
3024	/* Enable Pause-Loop exiting. */
3025	if ( pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT
3026	&& pVM->hm.s.vmx.cPleGapTicks
3027	&& pVM->hm.s.vmx.cPleWindowTicks)
3028	{
3029	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3030
3031	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3032	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3033	AssertRCReturn(rc, rc);
3034	}
3035
3036	if ((fVal & fZap) != fVal)
3037	{
3038	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3039	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3040	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3041	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3042	}
3043
3044	/* Commit it to the VMCS and update our cache. */
3045	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3046	AssertRCReturn(rc, rc);
3047	pVmcsInfo->u32ProcCtls2 = fVal;
3048
3049	return VINF_SUCCESS;
3050	}
3051
3052
3053	/**
3054	* Sets up processor-based VM-execution controls in the VMCS.
3055	*
3056	* @returns VBox status code.
3057	* @param pVCpu The cross context virtual CPU structure.
3058	* @param pVmcsInfo The VMCS info. object.
3059	*/
3060	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3061	{
3062	PVM pVM = pVCpu->CTX_SUFF(pVM);
3063
3064	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3065	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3066
3067	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3068	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3069	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3070	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3071	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3072	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3073	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3074
3075	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3076	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3077	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3078	{
3079	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3080	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3081	}
3082
3083	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3084	if (!pVM->hm.s.fNestedPaging)
3085	{
3086	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3087	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3088	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3089	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3090	}
3091
3092	/* Use TPR shadowing if supported by the CPU. */
3093	if ( PDMHasApic(pVM)
3094	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3095	{
3096	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3097	/* CR8 writes cause a VM-exit based on TPR threshold. */
3098	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3099	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3100	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3101	AssertRCReturn(rc, rc);
3102	}
3103	else
3104	{
3105	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3106	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3107	if (pVM->hm.s.fAllow64BitGuests)
3108	{
3109	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3110	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3111	}
3112	}
3113
3114	/* Use MSR-bitmaps if supported by the CPU. */
3115	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3116	{
3117	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3118	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3119	AssertRCReturn(rc, rc);
3120	}
3121
3122	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3123	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3124	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3125
3126	if ((fVal & fZap) != fVal)
3127	{
3128	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3129	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3130	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3131	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3132	}
3133
3134	/* Commit it to the VMCS and update our cache. */
3135	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3136	AssertRCReturn(rc, rc);
3137	pVmcsInfo->u32ProcCtls = fVal;
3138
3139	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3140	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo, false /* fIsNstGstVmcs */);
3141
3142	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3143	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3144	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3145
3146	/* Sanity check, should not really happen. */
3147	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3148	{ /* likely */ }
3149	else
3150	{
3151	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3152	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3153	}
3154
3155	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3156	return VINF_SUCCESS;
3157	}
3158
3159
3160	/**
3161	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3162	* Processor-based VM-execution) control fields in the VMCS.
3163	*
3164	* @returns VBox status code.
3165	* @param pVCpu The cross context virtual CPU structure.
3166	* @param pVmcsInfo The VMCS info. object.
3167	*/
3168	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3169	{
3170	/* Set the auto-load/store MSR area addresses in the VMCS. */
3171	int rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3172	if (RT_SUCCESS(rc))
3173	{
3174	/* Set the VMCS link pointer in the VMCS. */
3175	rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3176	if (RT_SUCCESS(rc))
3177	{
3178	/* Set the CR0/CR4 guest/host mask. */
3179	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3180	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3181	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3182	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3183	if (RT_SUCCESS(rc))
3184	{
3185	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3186	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3187	return VINF_SUCCESS;
3188	}
3189	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3190	}
3191	else
3192	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3193	}
3194	else
3195	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3196	return rc;
3197	}
3198
3199
3200	/**
3201	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3202	*
3203	* We shall setup those exception intercepts that don't change during the
3204	* lifetime of the VM here. The rest are done dynamically while loading the
3205	* guest state.
3206	*
3207	* @returns VBox status code.
3208	* @param pVCpu The cross context virtual CPU structure.
3209	* @param pVmcsInfo The VMCS info. object.
3210	*/
3211	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3212	{
3213	/*
3214	* The following exceptions are always intercepted:
3215	*
3216	* #AC - To prevent the guest from hanging the CPU.
3217	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3218	* recursive #DBs can cause a CPU hang.
3219	* #PF - To sync our shadow page tables when nested-paging is not used.
3220	*/
3221	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3222	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3223	\| RT_BIT(X86_XCPT_DB)
3224	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3225
3226	/* Commit it to the VMCS. */
3227	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3228	AssertRCReturn(rc, rc);
3229
3230	/* Update our cache of the exception bitmap. */
3231	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3232	return VINF_SUCCESS;
3233	}
3234
3235
3236	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3237	/**
3238	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3239	*
3240	* @returns VBox status code.
3241	* @param pVCpu The cross context virtual CPU structure.
3242	* @param pVmcsInfo The VMCS info. object.
3243	*/
3244	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3245	{
3246	PVM pVM = pVCpu->CTX_SUFF(pVM);
3247	int rc = hmR0VmxSetupVmcsLinkPtr(pVmcsInfo);
3248	if (RT_SUCCESS(rc))
3249	{
3250	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
3251	if (RT_SUCCESS(rc))
3252	{
3253	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3254	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
3255	if (RT_SUCCESS(rc))
3256	{
3257	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3258	rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3259	if (RT_SUCCESS(rc))
3260	return VINF_SUCCESS;
3261
3262	LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3263	}
3264	else
3265	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3266	}
3267	else
3268	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3269	}
3270	else
3271	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3272
3273	return rc;
3274	}
3275	#endif
3276
3277
3278	/**
3279	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3280	* VMX.
3281	*
3282	* @returns VBox status code.
3283	* @param pVCpu The cross context virtual CPU structure.
3284	* @param pVmcsInfo The VMCS info. object.
3285	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3286	*/
3287	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3288	{
3289	Assert(pVmcsInfo);
3290	Assert(pVmcsInfo->pvVmcs);
3291	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3292
3293	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3294	PVM pVM = pVCpu->CTX_SUFF(pVM);
3295	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3296	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3297
3298	LogFlowFunc(("\n"));
3299
3300	/*
3301	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3302	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3303	*/
3304	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3305	if (RT_SUCCESS(rc))
3306	{
3307	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3308	if (RT_SUCCESS(rc))
3309	{
3310	if (!fIsNstGstVmcs)
3311	{
3312	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3313	if (RT_SUCCESS(rc))
3314	{
3315	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
3316	if (RT_SUCCESS(rc))
3317	{
3318	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
3319	if (RT_SUCCESS(rc))
3320	{
3321	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
3322	if (RT_SUCCESS(rc))
3323	{ /* likely */ }
3324	else
3325	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
3326	}
3327	else
3328	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
3329	}
3330	else
3331	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
3332	}
3333	else
3334	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
3335	}
3336	else
3337	{
3338	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3339	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
3340	if (RT_SUCCESS(rc))
3341	{ /* likely */ }
3342	else
3343	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
3344	#else
3345	AssertFailed();
3346	#endif
3347	}
3348	}
3349	else
3350	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
3351	}
3352	else
3353	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
3354
3355	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
3356	if (RT_SUCCESS(rc))
3357	{
3358	rc = hmR0VmxClearVmcs(pVmcsInfo);
3359	if (RT_SUCCESS(rc))
3360	{ /* likely */ }
3361	else
3362	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
3363	}
3364
3365	/*
3366	* Update the last-error record both for failures and success, so we
3367	* can propagate the status code back to ring-3 for diagnostics.
3368	*/
3369	hmR0VmxUpdateErrorRecord(pVCpu, rc);
3370	NOREF(pszVmcs);
3371	return rc;
3372	}
3373
3374
3375	/**
3376	* Does global VT-x initialization (called during module initialization).
3377	*
3378	* @returns VBox status code.
3379	*/
3380	VMMR0DECL(int) VMXR0GlobalInit(void)
3381	{
3382	#ifdef HMVMX_USE_FUNCTION_TABLE
3383	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
3384	# ifdef VBOX_STRICT
3385	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
3386	Assert(g_apfnVMExitHandlers[i]);
3387	# endif
3388	#endif
3389	return VINF_SUCCESS;
3390	}
3391
3392
3393	/**
3394	* Does global VT-x termination (called during module termination).
3395	*/
3396	VMMR0DECL(void) VMXR0GlobalTerm()
3397	{
3398	/* Nothing to do currently. */
3399	}
3400
3401
3402	/**
3403	* Sets up and activates VT-x on the current CPU.
3404	*
3405	* @returns VBox status code.
3406	* @param pHostCpu The HM physical-CPU structure.
3407	* @param pVM The cross context VM structure. Can be
3408	* NULL after a host resume operation.
3409	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
3410	* fEnabledByHost is @c true).
3411	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
3412	* @a fEnabledByHost is @c true).
3413	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
3414	* enable VT-x on the host.
3415	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
3416	*/
3417	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
3418	PCSUPHWVIRTMSRS pHwvirtMsrs)
3419	{
3420	Assert(pHostCpu);
3421	Assert(pHwvirtMsrs);
3422	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3423
3424	/* Enable VT-x if it's not already enabled by the host. */
3425	if (!fEnabledByHost)
3426	{
3427	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
3428	if (RT_FAILURE(rc))
3429	return rc;
3430	}
3431
3432	/*
3433	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
3434	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
3435	* invalidated when flushing by VPID.
3436	*/
3437	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3438	{
3439	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
3440	pHostCpu->fFlushAsidBeforeUse = false;
3441	}
3442	else
3443	pHostCpu->fFlushAsidBeforeUse = true;
3444
3445	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
3446	++pHostCpu->cTlbFlushes;
3447
3448	return VINF_SUCCESS;
3449	}
3450
3451
3452	/**
3453	* Deactivates VT-x on the current CPU.
3454	*
3455	* @returns VBox status code.
3456	* @param pvCpuPage Pointer to the VMXON region.
3457	* @param HCPhysCpuPage Physical address of the VMXON region.
3458	*
3459	* @remarks This function should never be called when SUPR0EnableVTx() or
3460	* similar was used to enable VT-x on the host.
3461	*/
3462	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
3463	{
3464	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
3465
3466	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3467	return hmR0VmxLeaveRootMode();
3468	}
3469
3470
3471	/**
3472	* Does per-VM VT-x initialization.
3473	*
3474	* @returns VBox status code.
3475	* @param pVM The cross context VM structure.
3476	*/
3477	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
3478	{
3479	LogFlowFunc(("pVM=%p\n", pVM));
3480
3481	int rc = hmR0VmxStructsAlloc(pVM);
3482	if (RT_FAILURE(rc))
3483	{
3484	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
3485	return rc;
3486	}
3487
3488	return VINF_SUCCESS;
3489	}
3490
3491
3492	/**
3493	* Does per-VM VT-x termination.
3494	*
3495	* @returns VBox status code.
3496	* @param pVM The cross context VM structure.
3497	*/
3498	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
3499	{
3500	LogFlowFunc(("pVM=%p\n", pVM));
3501
3502	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3503	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
3504	{
3505	Assert(pVM->hm.s.vmx.pvScratch);
3506	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
3507	}
3508	#endif
3509	hmR0VmxStructsFree(pVM);
3510	return VINF_SUCCESS;
3511	}
3512
3513
3514	/**
3515	* Sets up the VM for execution using hardware-assisted VMX.
3516	* This function is only called once per-VM during initialization.
3517	*
3518	* @returns VBox status code.
3519	* @param pVM The cross context VM structure.
3520	*/
3521	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
3522	{
3523	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
3524	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3525
3526	LogFlowFunc(("pVM=%p\n", pVM));
3527
3528	/*
3529	* At least verify if VMX is enabled, since we can't check if we're in
3530	* VMX root mode or not without causing a #GP.
3531	*/
3532	RTCCUINTREG const uHostCR4 = ASMGetCR4();
3533	if (RT_LIKELY(uHostCR4 & X86_CR4_VMXE))
3534	{ /* likely */ }
3535	else
3536	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
3537
3538	/*
3539	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
3540	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
3541	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
3542	*/
3543	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3544	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
3545	\|\| !pVM->hm.s.vmx.pRealModeTSS))
3546	{
3547	LogRelFunc(("Invalid real-on-v86 state.\n"));
3548	return VERR_INTERNAL_ERROR;
3549	}
3550
3551	/* Initialize these always, see hmR3InitFinalizeR0().*/
3552	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
3553	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
3554
3555	/* Setup the tagged-TLB flush handlers. */
3556	int rc = hmR0VmxSetupTaggedTlb(pVM);
3557	if (RT_FAILURE(rc))
3558	{
3559	LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
3560	return rc;
3561	}
3562
3563	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
3564	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
3565	#if HC_ARCH_BITS == 64
3566	if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
3567	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
3568	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
3569	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
3570	#endif
3571
3572	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3573	{
3574	PVMCPU pVCpu = &pVM->aCpus[idCpu];
3575	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
3576
3577	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
3578	if (RT_SUCCESS(rc))
3579	{
3580	#if HC_ARCH_BITS == 32
3581	hmR0VmxInitVmcsReadCache(pVCpu);
3582	#endif
3583	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3584	if (pVM->cpum.ro.GuestFeatures.fVmx)
3585	{
3586	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
3587	if (RT_SUCCESS(rc))
3588	{ /* likely */ }
3589	else
3590	{
3591	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
3592	return rc;
3593	}
3594	}
3595	#endif
3596	}
3597	else
3598	{
3599	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
3600	return rc;
3601	}
3602	}
3603
3604	return VINF_SUCCESS;
3605	}
3606
3607
3608	#if HC_ARCH_BITS == 32
3609	# ifdef VBOX_ENABLE_64_BITS_GUESTS
3610	/**
3611	* Check if guest state allows safe use of 32-bit switcher again.
3612	*
3613	* Segment bases and protected mode structures must be 32-bit addressable
3614	* because the 32-bit switcher will ignore high dword when writing these VMCS
3615	* fields. See @bugref{8432} for details.
3616	*
3617	* @returns true if safe, false if must continue to use the 64-bit switcher.
3618	* @param pCtx Pointer to the guest-CPU context.
3619	*
3620	* @remarks No-long-jump zone!!!
3621	*/
3622	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
3623	{
3624	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
3625	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
3626	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3627	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3628	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
3629	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3630	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
3631	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
3632	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3633	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3634
3635	/* All good, bases are 32-bit. */
3636	return true;
3637	}
3638	# endif /* VBOX_ENABLE_64_BITS_GUESTS */
3639
3640	# ifdef VBOX_STRICT
3641	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
3642	{
3643	switch (idxField)
3644	{
3645	case VMX_VMCS_GUEST_RIP:
3646	case VMX_VMCS_GUEST_RSP:
3647	case VMX_VMCS_GUEST_SYSENTER_EIP:
3648	case VMX_VMCS_GUEST_SYSENTER_ESP:
3649	case VMX_VMCS_GUEST_GDTR_BASE:
3650	case VMX_VMCS_GUEST_IDTR_BASE:
3651	case VMX_VMCS_GUEST_CS_BASE:
3652	case VMX_VMCS_GUEST_DS_BASE:
3653	case VMX_VMCS_GUEST_ES_BASE:
3654	case VMX_VMCS_GUEST_FS_BASE:
3655	case VMX_VMCS_GUEST_GS_BASE:
3656	case VMX_VMCS_GUEST_SS_BASE:
3657	case VMX_VMCS_GUEST_LDTR_BASE:
3658	case VMX_VMCS_GUEST_TR_BASE:
3659	case VMX_VMCS_GUEST_CR3:
3660	return true;
3661	}
3662	return false;
3663	}
3664
3665	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3666	{
3667	switch (idxField)
3668	{
3669	/* Read-only fields. */
3670	case VMX_VMCS_RO_EXIT_QUALIFICATION:
3671	return true;
3672	}
3673	/* Remaining readable fields should also be writable. */
3674	return hmR0VmxIsValidWriteField(idxField);
3675	}
3676	# endif /* VBOX_STRICT */
3677
3678
3679	/**
3680	* Executes the specified handler in 64-bit mode.
3681	*
3682	* @returns VBox status code (no informational status codes).
3683	* @param pVCpu The cross context virtual CPU structure.
3684	* @param enmOp The operation to perform.
3685	* @param cParams Number of parameters.
3686	* @param paParam Array of 32-bit parameters.
3687	*/
3688	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
3689	{
3690	PVM pVM = pVCpu->CTX_SUFF(pVM);
3691	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
3692	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
3693	Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
3694	Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
3695
3696	#ifdef VBOX_STRICT
3697	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
3698	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
3699
3700	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
3701	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
3702	#endif
3703
3704	/* Disable interrupts. */
3705	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
3706
3707	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
3708	RTCPUID idHostCpu = RTMpCpuId();
3709	CPUMR0SetLApic(pVCpu, idHostCpu);
3710	#endif
3711
3712	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
3713
3714	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3715	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3716
3717	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
3718	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3719	hmR0VmxClearVmcs(pVmcsInfo);
3720
3721	/* Leave VMX root mode and disable VMX. */
3722	VMXDisable();
3723	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3724
3725	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
3726	CPUMSetHyperEIP(pVCpu, enmOp);
3727	for (int i = (int)cParams - 1; i >= 0; i--)
3728	CPUMPushHyper(pVCpu, paParam[i]);
3729
3730	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
3731
3732	/* Call the switcher. */
3733	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
3734	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
3735
3736	/* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
3737	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
3738
3739	/* Re-enter VMX root mode. */
3740	int rc2 = VMXEnable(HCPhysCpuPage);
3741	if (RT_FAILURE(rc2))
3742	{
3743	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3744	ASMSetFlags(fOldEFlags);
3745	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
3746	return rc2;
3747	}
3748
3749	/* Restore the VMCS as the current VMCS. */
3750	rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
3751	AssertRC(rc2);
3752	Assert(!(ASMGetFlags() & X86_EFL_IF));
3753	ASMSetFlags(fOldEFlags);
3754	return rc;
3755	}
3756
3757
3758	/**
3759	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
3760	* supporting 64-bit guests.
3761	*
3762	* @returns VBox status code.
3763	* @param fResume Whether to VMLAUNCH or VMRESUME.
3764	* @param pCtx Pointer to the guest-CPU context.
3765	* @param pCache Pointer to the VMCS batch cache.
3766	* @param pVM The cross context VM structure.
3767	* @param pVCpu The cross context virtual CPU structure.
3768	*/
3769	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
3770	{
3771	NOREF(fResume);
3772
3773	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3774	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3775	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3776
3777	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3778	pCache->uPos = 1;
3779	pCache->interPD = PGMGetInterPaeCR3(pVM);
3780	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
3781	#endif
3782
3783	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3784	pCache->TestIn.HCPhysCpuPage = 0;
3785	pCache->TestIn.HCPhysVmcs = 0;
3786	pCache->TestIn.pCache = 0;
3787	pCache->TestOut.HCPhysVmcs = 0;
3788	pCache->TestOut.pCache = 0;
3789	pCache->TestOut.pCtx = 0;
3790	pCache->TestOut.eflags = 0;
3791	#else
3792	NOREF(pCache);
3793	#endif
3794
3795	uint32_t aParam[10];
3796	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
3797	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
3798	aParam[2] = RT_LO_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
3799	aParam[3] = RT_HI_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
3800	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
3801	aParam[5] = 0;
3802	aParam[6] = VM_RC_ADDR(pVM, pVM);
3803	aParam[7] = 0;
3804	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
3805	aParam[9] = 0;
3806
3807	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3808	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
3809	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
3810	#endif
3811	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
3812
3813	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3814	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
3815	Assert(pCtx->dr[4] == 10);
3816	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
3817	#endif
3818
3819	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3820	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
3821	AssertMsg(pCache->TestIn.HCPhysVmcs == pVmcsInfo->HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3822	pVmcsInfo->HCPhysVmcs));
3823	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3824	pCache->TestOut.HCPhysVmcs));
3825	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
3826	pCache->TestOut.pCache));
3827	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
3828	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
3829	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
3830	pCache->TestOut.pCtx));
3831	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
3832	#endif
3833	NOREF(pCtx);
3834	return rc;
3835	}
3836	#endif
3837
3838
3839	/**
3840	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
3841	* the VMCS.
3842	*
3843	* @returns VBox status code.
3844	*/
3845	static int hmR0VmxExportHostControlRegs(void)
3846	{
3847	RTCCUINTREG uReg = ASMGetCR0();
3848	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
3849	AssertRCReturn(rc, rc);
3850
3851	uReg = ASMGetCR3();
3852	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
3853	AssertRCReturn(rc, rc);
3854
3855	uReg = ASMGetCR4();
3856	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
3857	AssertRCReturn(rc, rc);
3858	return rc;
3859	}
3860
3861
3862	/**
3863	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
3864	* the host-state area in the VMCS.
3865	*
3866	* @returns VBox status code.
3867	* @param pVCpu The cross context virtual CPU structure.
3868	*/
3869	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
3870	{
3871	#if HC_ARCH_BITS == 64
3872	/**
3873	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
3874	* requirements. See hmR0VmxExportHostSegmentRegs().
3875	*/
3876	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
3877	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
3878	{ \
3879	bool fValidSelector = true; \
3880	if ((selValue) & X86_SEL_LDT) \
3881	{ \
3882	uint32_t uAttr = ASMGetSegAttr((selValue)); \
3883	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
3884	} \
3885	if (fValidSelector) \
3886	{ \
3887	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
3888	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
3889	} \
3890	(selValue) = 0; \
3891	}
3892
3893	/*
3894	* If we've executed guest code using hardware-assisted VMX, the host-state bits
3895	* will be messed up. We should -not- save the messed up state without restoring
3896	* the original host-state, see @bugref{7240}.
3897	*
3898	* This apparently can happen (most likely the FPU changes), deal with it rather than
3899	* asserting. Was observed booting Solaris 10u10 32-bit guest.
3900	*/
3901	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
3902	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
3903	{
3904	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
3905	pVCpu->idCpu));
3906	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
3907	}
3908	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
3909	#else
3910	RT_NOREF(pVCpu);
3911	#endif
3912
3913	/*
3914	* Host DS, ES, FS and GS segment registers.
3915	*/
3916	#if HC_ARCH_BITS == 64
3917	RTSEL uSelDS = ASMGetDS();
3918	RTSEL uSelES = ASMGetES();
3919	RTSEL uSelFS = ASMGetFS();
3920	RTSEL uSelGS = ASMGetGS();
3921	#else
3922	RTSEL uSelDS = 0;
3923	RTSEL uSelES = 0;
3924	RTSEL uSelFS = 0;
3925	RTSEL uSelGS = 0;
3926	#endif
3927
3928	/*
3929	* Host CS and SS segment registers.
3930	*/
3931	RTSEL uSelCS = ASMGetCS();
3932	RTSEL uSelSS = ASMGetSS();
3933
3934	/*
3935	* Host TR segment register.
3936	*/
3937	RTSEL uSelTR = ASMGetTR();
3938
3939	#if HC_ARCH_BITS == 64
3940	/*
3941	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
3942	* gain VM-entry and restore them before we get preempted.
3943	*
3944	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
3945	*/
3946	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3947	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3948	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3949	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3950	# undef VMXLOCAL_ADJUST_HOST_SEG
3951	#endif
3952
3953	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
3954	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
3955	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
3956	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
3957	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
3958	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
3959	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
3960	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
3961	Assert(uSelCS);
3962	Assert(uSelTR);
3963
3964	/* Write these host selector fields into the host-state area in the VMCS. */
3965	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3966	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3967	#if HC_ARCH_BITS == 64
3968	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3969	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3970	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3971	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3972	#else
3973	NOREF(uSelDS);
3974	NOREF(uSelES);
3975	NOREF(uSelFS);
3976	NOREF(uSelGS);
3977	#endif
3978	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3979	AssertRCReturn(rc, rc);
3980
3981	/*
3982	* Host GDTR and IDTR.
3983	*/
3984	RTGDTR Gdtr;
3985	RTIDTR Idtr;
3986	RT_ZERO(Gdtr);
3987	RT_ZERO(Idtr);
3988	ASMGetGDTR(&Gdtr);
3989	ASMGetIDTR(&Idtr);
3990	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3991	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3992	AssertRCReturn(rc, rc);
3993
3994	#if HC_ARCH_BITS == 64
3995	/*
3996	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
3997	* them to the maximum limit (0xffff) on every VM-exit.
3998	*/
3999	if (Gdtr.cbGdt != 0xffff)
4000	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4001
4002	/*
4003	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4004	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4005	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4006	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4007	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4008	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4009	* at 0xffff on hosts where we are sure it won't cause trouble.
4010	*/
4011	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4012	if (Idtr.cbIdt < 0x0fff)
4013	# else
4014	if (Idtr.cbIdt != 0xffff)
4015	# endif
4016	{
4017	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4018	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4019	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4020	}
4021	#endif
4022
4023	/*
4024	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4025	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4026	* RPL should be too in most cases.
4027	*/
4028	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4029	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4030
4031	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4032	#if HC_ARCH_BITS == 64
4033	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4034
4035	/*
4036	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4037	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4038	* restoration if the host has something else. Task switching is not supported in 64-bit
4039	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4040	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4041	*
4042	* [1] See Intel spec. 3.5 "System Descriptor Types".
4043	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4044	*/
4045	PVM pVM = pVCpu->CTX_SUFF(pVM);
4046	Assert(pDesc->System.u4Type == 11);
4047	if ( pDesc->System.u16LimitLow != 0x67
4048	\|\| pDesc->System.u4LimitHigh)
4049	{
4050	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4051	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4052	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4053	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4054	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4055	}
4056
4057	/*
4058	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4059	*/
4060	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4061	{
4062	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4063	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4064	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4065	{
4066	/* The GDT is read-only but the writable GDT is available. */
4067	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4068	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4069	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4070	AssertRCReturn(rc, rc);
4071	}
4072	}
4073	#else
4074	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4075	#endif
4076	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4077	AssertRCReturn(rc, rc);
4078
4079	/*
4080	* Host FS base and GS base.
4081	*/
4082	#if HC_ARCH_BITS == 64
4083	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4084	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4085	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4086	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4087	AssertRCReturn(rc, rc);
4088
4089	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4090	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4091	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4092	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4093	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4094	#endif
4095	return VINF_SUCCESS;
4096	}
4097
4098
4099	/**
4100	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4101	* host-state area of the VMCS.
4102	*
4103	* These MSRs will be automatically restored on the host after every successful
4104	* VM-exit.
4105	*
4106	* @returns VBox status code.
4107	* @param pVCpu The cross context virtual CPU structure.
4108	*
4109	* @remarks No-long-jump zone!!!
4110	*/
4111	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4112	{
4113	AssertPtr(pVCpu);
4114
4115	/*
4116	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4117	* rather than swapping them on every VM-entry.
4118	*/
4119	hmR0VmxLazySaveHostMsrs(pVCpu);
4120
4121	/*
4122	* Host Sysenter MSRs.
4123	*/
4124	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4125	#if HC_ARCH_BITS == 32
4126	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4127	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4128	#else
4129	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4130	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4131	#endif
4132	AssertRCReturn(rc, rc);
4133
4134	/*
4135	* Host EFER MSR.
4136	*
4137	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4138	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4139	*/
4140	PVM pVM = pVCpu->CTX_SUFF(pVM);
4141	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4142	{
4143	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4144	AssertRCReturn(rc, rc);
4145	}
4146
4147	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4148	* hmR0VmxExportGuestEntryExitCtls(). */
4149
4150	return VINF_SUCCESS;
4151	}
4152
4153
4154	/**
4155	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4156	*
4157	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4158	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4159	*
4160	* @returns true if we need to load guest EFER, false otherwise.
4161	* @param pVCpu The cross context virtual CPU structure.
4162	*
4163	* @remarks Requires EFER, CR4.
4164	* @remarks No-long-jump zone!!!
4165	*/
4166	static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu)
4167	{
4168	#ifdef HMVMX_ALWAYS_SWAP_EFER
4169	RT_NOREF(pVCpu);
4170	return true;
4171	#else
4172	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4173	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4174	/* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4175	if (CPUMIsGuestInLongModeEx(pCtx))
4176	return false;
4177	#endif
4178
4179	PVM pVM = pVCpu->CTX_SUFF(pVM);
4180	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4181	uint64_t const u64GuestEfer = pCtx->msrEFER;
4182
4183	/*
4184	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4185	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4186	*/
4187	if ( CPUMIsGuestInLongModeEx(pCtx)
4188	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4189	return true;
4190
4191	/*
4192	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4193	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4194	*
4195	* See Intel spec. 4.5 "IA-32e Paging".
4196	* See Intel spec. 4.1.1 "Three Paging Modes".
4197	*
4198	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4199	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4200	*/
4201	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4202	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4203	if ( (pCtx->cr4 & X86_CR4_PAE)
4204	&& (pCtx->cr0 & X86_CR0_PG)
4205	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4206	{
4207	/* Assert that host is NX capable. */
4208	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4209	return true;
4210	}
4211
4212	return false;
4213	#endif
4214	}
4215
4216	/**
4217	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4218	* VMCS.
4219	*
4220	* This is typically required when the guest changes paging mode.
4221	*
4222	* @returns VBox status code.
4223	* @param pVCpu The cross context virtual CPU structure.
4224	* @param pVmxTransient The VMX-transient structure.
4225	*
4226	* @remarks Requires EFER.
4227	* @remarks No-long-jump zone!!!
4228	*/
4229	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4230	{
4231	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4232	{
4233	PVM pVM = pVCpu->CTX_SUFF(pVM);
4234	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4235
4236	/*
4237	* VM-entry controls.
4238	*/
4239	{
4240	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4241	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4242
4243	/*
4244	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4245	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4246	*
4247	* For nested-guests, this is a mandatory VM-entry control. It's also
4248	* required because we do not want to leak host bits to the nested-guest.
4249	*/
4250	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4251
4252	/*
4253	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4254	*
4255	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4256	* required to get the nested-guest working with hardware-assisted VMX execution.
4257	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested-hypervisor
4258	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4259	* here rather than while merging the guest VMCS controls.
4260	*/
4261	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4262	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4263	else
4264	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4265
4266	/*
4267	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4268	*
4269	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4270	* regardless of whether the nested-guest VMCS specifies it because we are free to
4271	* load whatever MSRs we require and we do not need to modify the guest visible copy
4272	* of the VM-entry MSR load area.
4273	*/
4274	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4275	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4276	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4277	else
4278	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4279
4280	/*
4281	* The following should -not- be set (since we're not in SMM mode):
4282	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4283	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4284	*/
4285
4286	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4287	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4288
4289	if ((fVal & fZap) == fVal)
4290	{ /* likely */ }
4291	else
4292	{
4293	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4294	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4295	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4296	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4297	}
4298
4299	/* Commit it to the VMCS. */
4300	if (pVmcsInfo->u32EntryCtls != fVal)
4301	{
4302	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4303	AssertRCReturn(rc, rc);
4304	pVmcsInfo->u32EntryCtls = fVal;
4305	}
4306	}
4307
4308	/*
4309	* VM-exit controls.
4310	*/
4311	{
4312	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4313	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4314
4315	/*
4316	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4317	* supported the 1-setting of this bit.
4318	*
4319	* For nested-guests, we set the "save debug controls" as the converse
4320	* "load debug controls" is mandatory for nested-guests anyway.
4321	*/
4322	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4323
4324	/*
4325	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4326	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4327	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4328	* hmR0VmxExportHostMsrs().
4329	*
4330	* For nested-guests, we always set this bit as we do not support 32-bit
4331	* hosts.
4332	*/
4333	#if HC_ARCH_BITS == 64
4334	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4335	#else
4336	Assert(!pVmxTransient->fIsNestedGuest);
4337	Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
4338	\|\| pVmcsInfo->pfnStartVM == VMXR0StartVM32);
4339	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
4340	if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
4341	{
4342	/* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
4343	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4344	}
4345	else
4346	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
4347	#endif
4348
4349	/*
4350	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4351	*
4352	* For nested-guests, we should use the "save IA32_EFER" control if we also
4353	* used the "load IA32_EFER" control while exporting VM-entry controls.
4354	*/
4355	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4356	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4357	{
4358	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4359	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4360	}
4361
4362	/*
4363	* Enable saving of the VMX-preemption timer value on VM-exit.
4364	* For nested-guests, currently not exposed/used.
4365	*/
4366	if ( pVM->hm.s.vmx.fUsePreemptTimer
4367	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4368	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4369
4370	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4371	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4372
4373	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4374	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4375	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4376
4377	if ((fVal & fZap) == fVal)
4378	{ /* likely */ }
4379	else
4380	{
4381	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4382	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4383	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4384	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4385	}
4386
4387	/* Commit it to the VMCS. */
4388	if (pVmcsInfo->u32ExitCtls != fVal)
4389	{
4390	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4391	AssertRCReturn(rc, rc);
4392	pVmcsInfo->u32ExitCtls = fVal;
4393	}
4394	}
4395
4396	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4397	}
4398	return VINF_SUCCESS;
4399	}
4400
4401
4402	/**
4403	* Sets the TPR threshold in the VMCS.
4404	*
4405	* @returns VBox status code.
4406	* @param pVCpu The cross context virtual CPU structure.
4407	* @param pVmcsInfo The VMCS info. object.
4408	* @param u32TprThreshold The TPR threshold (task-priority class only).
4409	*/
4410	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4411	{
4412	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4413	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4414	RT_NOREF2(pVCpu, pVmcsInfo);
4415	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4416	}
4417
4418
4419	/**
4420	* Exports the guest APIC TPR state into the VMCS.
4421	*
4422	* @returns VBox status code.
4423	* @param pVCpu The cross context virtual CPU structure.
4424	* @param pVmxTransient The VMX-transient structure.
4425	*
4426	* @remarks No-long-jump zone!!!
4427	*/
4428	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4429	{
4430	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4431	{
4432	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4433
4434	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4435	if (!pVmxTransient->fIsNestedGuest)
4436	{
4437	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4438	&& APICIsEnabled(pVCpu))
4439	{
4440	/*
4441	* Setup TPR shadowing.
4442	*/
4443	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4444	{
4445	bool fPendingIntr = false;
4446	uint8_t u8Tpr = 0;
4447	uint8_t u8PendingIntr = 0;
4448	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4449	AssertRCReturn(rc, rc);
4450
4451	/*
4452	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4453	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4454	* priority of the pending interrupt so we can deliver the interrupt. If there
4455	* are no interrupts pending, set threshold to 0 to not cause any
4456	* TPR-below-threshold VM-exits.
4457	*/
4458	Assert(pVmcsInfo->pbVirtApic);
4459	pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
4460	uint32_t u32TprThreshold = 0;
4461	if (fPendingIntr)
4462	{
4463	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4464	(which is the Task-Priority Class). */
4465	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4466	const uint8_t u8TprPriority = u8Tpr >> 4;
4467	if (u8PendingPriority <= u8TprPriority)
4468	u32TprThreshold = u8PendingPriority;
4469	}
4470
4471	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
4472	AssertRCReturn(rc, rc);
4473	}
4474	}
4475	}
4476	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4477	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4478	}
4479	return VINF_SUCCESS;
4480	}
4481
4482
4483	/**
4484	* Gets the guest interruptibility-state.
4485	*
4486	* @returns Guest's interruptibility-state.
4487	* @param pVCpu The cross context virtual CPU structure.
4488	* @param pVmcsInfo The VMCS info. object.
4489	*
4490	* @remarks No-long-jump zone!!!
4491	*/
4492	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
4493	{
4494	/*
4495	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4496	*/
4497	uint32_t fIntrState = 0;
4498	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4499	{
4500	/* If inhibition is active, RIP and RFLAGS should've been updated
4501	(i.e. read previously from the VMCS or from ring-3). */
4502	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4503	#ifdef VBOX_STRICT
4504	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
4505	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4506	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
4507	#endif
4508	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
4509	{
4510	if (pCtx->eflags.Bits.u1IF)
4511	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4512	else
4513	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
4514	}
4515	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4516	{
4517	/*
4518	* We can clear the inhibit force flag as even if we go back to the recompiler
4519	* without executing guest code in VT-x, the flag's condition to be cleared is
4520	* met and thus the cleared state is correct.
4521	*/
4522	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4523	}
4524	}
4525
4526	/*
4527	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
4528	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
4529	* setting this would block host-NMIs and IRET will not clear the blocking.
4530	*
4531	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
4532	*
4533	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
4534	*/
4535	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
4536	&& CPUMIsGuestNmiBlocking(pVCpu))
4537	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
4538
4539	return fIntrState;
4540	}
4541
4542
4543	/**
4544	* Exports the exception intercepts required for guest execution in the VMCS.
4545	*
4546	* @returns VBox status code.
4547	* @param pVCpu The cross context virtual CPU structure.
4548	* @param pVmxTransient The VMX-transient structure.
4549	*
4550	* @remarks No-long-jump zone!!!
4551	*/
4552	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4553	{
4554	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
4555	{
4556	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
4557	if ( !pVmxTransient->fIsNestedGuest
4558	&& pVCpu->hm.s.fGIMTrapXcptUD)
4559	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
4560	else
4561	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
4562
4563	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
4564	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
4565	}
4566	return VINF_SUCCESS;
4567	}
4568
4569
4570	/**
4571	* Exports the guest's RIP into the guest-state area in the VMCS.
4572	*
4573	* @returns VBox status code.
4574	* @param pVCpu The cross context virtual CPU structure.
4575	*
4576	* @remarks No-long-jump zone!!!
4577	*/
4578	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
4579	{
4580	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4581	{
4582	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
4583
4584	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
4585	AssertRCReturn(rc, rc);
4586
4587	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
4588	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
4589	}
4590	return VINF_SUCCESS;
4591	}
4592
4593
4594	/**
4595	* Exports the guest's RSP into the guest-state area in the VMCS.
4596	*
4597	* @returns VBox status code.
4598	* @param pVCpu The cross context virtual CPU structure.
4599	*
4600	* @remarks No-long-jump zone!!!
4601	*/
4602	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
4603	{
4604	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
4605	{
4606	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
4607
4608	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
4609	AssertRCReturn(rc, rc);
4610
4611	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
4612	}
4613	return VINF_SUCCESS;
4614	}
4615
4616
4617	/**
4618	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
4619	*
4620	* @returns VBox status code.
4621	* @param pVCpu The cross context virtual CPU structure.
4622	* @param pVmxTransient The VMX-transient structure.
4623	*
4624	* @remarks No-long-jump zone!!!
4625	*/
4626	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4627	{
4628	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
4629	{
4630	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
4631
4632	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
4633	Let us assert it as such and use 32-bit VMWRITE. */
4634	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
4635	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
4636	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
4637	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
4638
4639	/*
4640	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
4641	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
4642	* can run the real-mode guest code under Virtual 8086 mode.
4643	*/
4644	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4645	if (pVmcsInfo->RealMode.fRealOnV86Active)
4646	{
4647	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4648	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4649	Assert(!pVmxTransient->fIsNestedGuest);
4650	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
4651	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
4652	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
4653	}
4654
4655	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
4656	AssertRCReturn(rc, rc);
4657
4658	/*
4659	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
4660	*
4661	* We must avoid setting any automatic debug exceptions delivery when single-stepping
4662	* through the hypervisor debugger using EFLAGS.TF.
4663	*/
4664	if ( !pVmxTransient->fIsNestedGuest
4665	&& !pVCpu->hm.s.fSingleInstruction
4666	&& fEFlags.Bits.u1TF)
4667	{
4668	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
4669	* premature trips to ring-3 esp since IEM does not yet handle it. */
4670	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
4671	AssertRCReturn(rc, rc);
4672	}
4673	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
4674	* nested-guest VMCS. */
4675
4676	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
4677	Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
4678	}
4679	return VINF_SUCCESS;
4680	}
4681
4682
4683	/**
4684	* Exports the guest CR0 control register into the guest-state area in the VMCS.
4685	*
4686	* The guest FPU state is always pre-loaded hence we don't need to bother about
4687	* sharing FPU related CR0 bits between the guest and host.
4688	*
4689	* @returns VBox status code.
4690	* @param pVCpu The cross context virtual CPU structure.
4691	* @param pVmxTransient The VMX-transient structure.
4692	*
4693	* @remarks No-long-jump zone!!!
4694	*/
4695	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4696	{
4697	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
4698	{
4699	PVM pVM = pVCpu->CTX_SUFF(pVM);
4700	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4701
4702	/*
4703	* Figure out fixed CR0 bits in VMX operation.
4704	*/
4705	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4706	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4707	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4708	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
4709	else
4710	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
4711
4712	if (!pVmxTransient->fIsNestedGuest)
4713	{
4714	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4715	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4716	uint64_t const u64ShadowCr0 = u64GuestCr0;
4717	Assert(!RT_HI_U32(u64GuestCr0));
4718
4719	/*
4720	* Setup VT-x's view of the guest CR0.
4721	*/
4722	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
4723	if (pVM->hm.s.fNestedPaging)
4724	{
4725	if (CPUMIsGuestPagingEnabled(pVCpu))
4726	{
4727	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
4728	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
4729	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
4730	}
4731	else
4732	{
4733	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
4734	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
4735	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
4736	}
4737
4738	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
4739	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4740	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
4741	}
4742	else
4743	{
4744	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
4745	u64GuestCr0 \|= X86_CR0_WP;
4746	}
4747
4748	/*
4749	* Guest FPU bits.
4750	*
4751	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
4752	* using CR0.TS.
4753	*
4754	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
4755	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
4756	*/
4757	u64GuestCr0 \|= X86_CR0_NE;
4758
4759	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
4760	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
4761
4762	/*
4763	* Update exception intercepts.
4764	*/
4765	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
4766	if (pVmcsInfo->RealMode.fRealOnV86Active)
4767	{
4768	Assert(PDMVmmDevHeapIsEnabled(pVM));
4769	Assert(pVM->hm.s.vmx.pRealModeTSS);
4770	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
4771	}
4772	else
4773	{
4774	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
4775	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
4776	if (fInterceptMF)
4777	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
4778	}
4779
4780	/* Additional intercepts for debugging, define these yourself explicitly. */
4781	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4782	uXcptBitmap \|= 0
4783	\| RT_BIT(X86_XCPT_BP)
4784	\| RT_BIT(X86_XCPT_DE)
4785	\| RT_BIT(X86_XCPT_NM)
4786	\| RT_BIT(X86_XCPT_TS)
4787	\| RT_BIT(X86_XCPT_UD)
4788	\| RT_BIT(X86_XCPT_NP)
4789	\| RT_BIT(X86_XCPT_SS)
4790	\| RT_BIT(X86_XCPT_GP)
4791	\| RT_BIT(X86_XCPT_PF)
4792	\| RT_BIT(X86_XCPT_MF)
4793	;
4794	#elif defined(HMVMX_ALWAYS_TRAP_PF)
4795	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
4796	#endif
4797	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
4798	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
4799	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
4800
4801	/* Apply the fixed CR0 bits and enable caching. */
4802	u64GuestCr0 \|= fSetCr0;
4803	u64GuestCr0 &= fZapCr0;
4804	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4805
4806	/* Commit the CR0 and related fields to the guest VMCS. */
4807	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCr0);
4808	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4809	if (uProcCtls != pVmcsInfo->u32ProcCtls)
4810	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4811	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
4812	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
4813	AssertRCReturn(rc, rc);
4814
4815	/* Update our caches. */
4816	pVmcsInfo->u32ProcCtls = uProcCtls;
4817	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
4818
4819	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4820	}
4821	else
4822	{
4823	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
4824	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4825	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4826	uint64_t const u64ShadowCr0 = pVmcsNstGst->u64Cr0ReadShadow.u;
4827	Assert(!RT_HI_U32(u64GuestCr0));
4828	Assert(u64GuestCr0 & X86_CR0_NE);
4829
4830	/* Apply the fixed CR0 bits and enable caching. */
4831	u64GuestCr0 \|= fSetCr0;
4832	u64GuestCr0 &= fZapCr0;
4833	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4834
4835	/* Commit the CR0 and CR0 read shadow to the nested-guest VMCS. */
4836	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCr0);
4837	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4838	AssertRCReturn(rc, rc);
4839
4840	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4841	}
4842
4843	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
4844	}
4845
4846	return VINF_SUCCESS;
4847	}
4848
4849
4850	/**
4851	* Exports the guest control registers (CR3, CR4) into the guest-state area
4852	* in the VMCS.
4853	*
4854	* @returns VBox strict status code.
4855	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
4856	* without unrestricted guest access and the VMMDev is not presently
4857	* mapped (e.g. EFI32).
4858	*
4859	* @param pVCpu The cross context virtual CPU structure.
4860	* @param pVmxTransient The VMX-transient structure.
4861	*
4862	* @remarks No-long-jump zone!!!
4863	*/
4864	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4865	{
4866	int rc = VINF_SUCCESS;
4867	PVM pVM = pVCpu->CTX_SUFF(pVM);
4868
4869	/*
4870	* Guest CR2.
4871	* It's always loaded in the assembler code. Nothing to do here.
4872	*/
4873
4874	/*
4875	* Guest CR3.
4876	*/
4877	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
4878	{
4879	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
4880
4881	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
4882	if (pVM->hm.s.fNestedPaging)
4883	{
4884	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4885	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
4886
4887	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
4888	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
4889	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
4890	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
4891
4892	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
4893	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
4894	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
4895
4896	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
4897	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
4898	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
4899	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
4900	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
4901	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
4902	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
4903
4904	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
4905	AssertRCReturn(rc, rc);
4906
4907	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4908	if ( pVM->hm.s.vmx.fUnrestrictedGuest
4909	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
4910	{
4911	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
4912	if (CPUMIsGuestInPAEModeEx(pCtx))
4913	{
4914	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
4915	AssertRCReturn(rc, rc);
4916	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
4917	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
4918	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
4919	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
4920	AssertRCReturn(rc, rc);
4921	}
4922
4923	/*
4924	* The guest's view of its CR3 is unblemished with nested paging when the
4925	* guest is using paging or we have unrestricted guest execution to handle
4926	* the guest when it's not using paging.
4927	*/
4928	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
4929	GCPhysGuestCR3 = pCtx->cr3;
4930	}
4931	else
4932	{
4933	/*
4934	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
4935	* thinks it accesses physical memory directly, we use our identity-mapped
4936	* page table to map guest-linear to guest-physical addresses. EPT takes care
4937	* of translating it to host-physical addresses.
4938	*/
4939	RTGCPHYS GCPhys;
4940	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4941
4942	/* We obtain it here every time as the guest could have relocated this PCI region. */
4943	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4944	if (RT_SUCCESS(rc))
4945	{ /* likely */ }
4946	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4947	{
4948	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
4949	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4950	}
4951	else
4952	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4953
4954	GCPhysGuestCR3 = GCPhys;
4955	}
4956
4957	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
4958	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4959	AssertRCReturn(rc, rc);
4960	}
4961	else
4962	{
4963	/* Non-nested paging case, just use the hypervisor's CR3. */
4964	RTHCPHYS const HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4965
4966	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
4967	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4968	AssertRCReturn(rc, rc);
4969	}
4970
4971	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
4972	}
4973
4974	/*
4975	* Guest CR4.
4976	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4977	*/
4978	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
4979	{
4980	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4981	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4982	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
4983
4984	/*
4985	* Figure out fixed CR4 bits in VMX operation.
4986	*/
4987	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
4988	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
4989
4990	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
4991	uint64_t u64GuestCr4 = pCtx->cr4;
4992	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest ? pCtx->cr4 : pVmcsNstGst->u64Cr4ReadShadow.u;
4993	Assert(!RT_HI_U32(u64GuestCr4));
4994
4995	/*
4996	* Setup VT-x's view of the guest CR4.
4997	*
4998	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
4999	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5000	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5001	*
5002	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5003	*/
5004	if (pVmcsInfo->RealMode.fRealOnV86Active)
5005	{
5006	Assert(pVM->hm.s.vmx.pRealModeTSS);
5007	Assert(PDMVmmDevHeapIsEnabled(pVM));
5008	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5009	}
5010
5011	if (pVM->hm.s.fNestedPaging)
5012	{
5013	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5014	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5015	{
5016	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5017	u64GuestCr4 \|= X86_CR4_PSE;
5018	/* Our identity mapping is a 32-bit page directory. */
5019	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5020	}
5021	/* else use guest CR4.*/
5022	}
5023	else
5024	{
5025	Assert(!pVmxTransient->fIsNestedGuest);
5026
5027	/*
5028	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5029	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5030	*/
5031	switch (pVCpu->hm.s.enmShadowMode)
5032	{
5033	case PGMMODE_REAL: /* Real-mode. */
5034	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5035	case PGMMODE_32_BIT: /* 32-bit paging. */
5036	{
5037	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5038	break;
5039	}
5040
5041	case PGMMODE_PAE: /* PAE paging. */
5042	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5043	{
5044	u64GuestCr4 \|= X86_CR4_PAE;
5045	break;
5046	}
5047
5048	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5049	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5050	#ifdef VBOX_ENABLE_64_BITS_GUESTS
5051	break;
5052	#endif
5053	default:
5054	AssertFailed();
5055	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5056	}
5057	}
5058
5059	/* Apply the fixed CR4 bits (mainly CR4.VMXE). */
5060	u64GuestCr4 \|= fSetCr4;
5061	u64GuestCr4 &= fZapCr4;
5062
5063	/* Commit the CR4 and CR4 read shadow to the guest VMCS. */
5064	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR4, u64GuestCr4);
5065	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
5066	AssertRCReturn(rc, rc);
5067
5068	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5069	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5070
5071	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5072
5073	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5074	}
5075	return rc;
5076	}
5077
5078
5079	/**
5080	* Exports the guest debug registers into the guest-state area in the VMCS.
5081	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5082	*
5083	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5084	*
5085	* @returns VBox status code.
5086	* @param pVCpu The cross context virtual CPU structure.
5087	* @param pVmxTransient The VMX-transient structure.
5088	*
5089	* @remarks No-long-jump zone!!!
5090	*/
5091	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5092	{
5093	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5094
5095	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5096	* stepping. */
5097	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5098	if (pVmxTransient->fIsNestedGuest)
5099	{
5100	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5101	AssertRCReturn(rc, rc);
5102	return VINF_SUCCESS;
5103	}
5104
5105	#ifdef VBOX_STRICT
5106	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5107	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5108	{
5109	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5110	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5111	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5112	}
5113	#endif
5114
5115	bool fSteppingDB = false;
5116	bool fInterceptMovDRx = false;
5117	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5118	if (pVCpu->hm.s.fSingleInstruction)
5119	{
5120	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5121	PVM pVM = pVCpu->CTX_SUFF(pVM);
5122	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5123	{
5124	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5125	Assert(fSteppingDB == false);
5126	}
5127	else
5128	{
5129	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5130	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5131	pVCpu->hm.s.fClearTrapFlag = true;
5132	fSteppingDB = true;
5133	}
5134	}
5135
5136	uint32_t u32GuestDr7;
5137	if ( fSteppingDB
5138	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5139	{
5140	/*
5141	* Use the combined guest and host DRx values found in the hypervisor register set
5142	* because the hypervisor debugger has breakpoints active or someone is single stepping
5143	* on the host side without a monitor trap flag.
5144	*
5145	* Note! DBGF expects a clean DR6 state before executing guest code.
5146	*/
5147	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5148	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5149	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
5150	{
5151	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5152	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
5153	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
5154	}
5155	else
5156	#endif
5157	if (!CPUMIsHyperDebugStateActive(pVCpu))
5158	{
5159	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5160	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5161	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5162	}
5163
5164	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5165	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
5166	pVCpu->hm.s.fUsingHyperDR7 = true;
5167	fInterceptMovDRx = true;
5168	}
5169	else
5170	{
5171	/*
5172	* If the guest has enabled debug registers, we need to load them prior to
5173	* executing guest code so they'll trigger at the right time.
5174	*/
5175	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5176	{
5177	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5178	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5179	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
5180	{
5181	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5182	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
5183	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
5184	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5185	}
5186	else
5187	#endif
5188	if (!CPUMIsGuestDebugStateActive(pVCpu))
5189	{
5190	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5191	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5192	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5193	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5194	}
5195	Assert(!fInterceptMovDRx);
5196	}
5197	/*
5198	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5199	* must intercept #DB in order to maintain a correct DR6 guest value, and
5200	* because we need to intercept it to prevent nested #DBs from hanging the
5201	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5202	*/
5203	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5204	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
5205	&& !CPUMIsGuestDebugStateActive(pVCpu))
5206	#else
5207	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5208	#endif
5209	{
5210	fInterceptMovDRx = true;
5211	}
5212
5213	/* Update DR7 with the actual guest value. */
5214	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5215	pVCpu->hm.s.fUsingHyperDR7 = false;
5216	}
5217
5218	if (fInterceptMovDRx)
5219	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5220	else
5221	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5222
5223	/*
5224	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5225	* monitor-trap flag and update our cache.
5226	*/
5227	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5228	{
5229	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5230	AssertRCReturn(rc2, rc2);
5231	pVmcsInfo->u32ProcCtls = uProcCtls;
5232	}
5233
5234	/*
5235	* Update guest DR7.
5236	*/
5237	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
5238	AssertRCReturn(rc, rc);
5239
5240	/*
5241	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
5242	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
5243	*
5244	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5245	*/
5246	if (fSteppingDB)
5247	{
5248	Assert(pVCpu->hm.s.fSingleInstruction);
5249	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
5250
5251	uint32_t fIntrState = 0;
5252	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
5253	AssertRCReturn(rc, rc);
5254
5255	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5256	{
5257	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5258	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5259	AssertRCReturn(rc, rc);
5260	}
5261	}
5262
5263	return VINF_SUCCESS;
5264	}
5265
5266
5267	#ifdef VBOX_STRICT
5268	/**
5269	* Strict function to validate segment registers.
5270	*
5271	* @param pVCpu The cross context virtual CPU structure.
5272	* @param pVmcsInfo The VMCS info. object.
5273	*
5274	* @remarks Will import guest CR0 on strict builds during validation of
5275	* segments.
5276	*/
5277	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
5278	{
5279	/*
5280	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5281	*
5282	* The reason we check for attribute value 0 in this function and not just the unusable bit is
5283	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
5284	* unusable bit and doesn't change the guest-context value.
5285	*/
5286	PVM pVM = pVCpu->CTX_SUFF(pVM);
5287	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5288	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
5289	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
5290	&& ( !CPUMIsGuestInRealModeEx(pCtx)
5291	&& !CPUMIsGuestInV86ModeEx(pCtx)))
5292	{
5293	/* Protected mode checks */
5294	/* CS */
5295	Assert(pCtx->cs.Attr.n.u1Present);
5296	Assert(!(pCtx->cs.Attr.u & 0xf00));
5297	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
5298	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5299	\|\| !(pCtx->cs.Attr.n.u1Granularity));
5300	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
5301	\|\| (pCtx->cs.Attr.n.u1Granularity));
5302	/* CS cannot be loaded with NULL in protected mode. */
5303	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
5304	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5305	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
5306	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5307	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
5308	else
5309	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
5310	/* SS */
5311	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5312	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
5313	if ( !(pCtx->cr0 & X86_CR0_PE)
5314	\|\| pCtx->cs.Attr.n.u4Type == 3)
5315	{
5316	Assert(!pCtx->ss.Attr.n.u2Dpl);
5317	}
5318	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
5319	{
5320	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5321	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
5322	Assert(pCtx->ss.Attr.n.u1Present);
5323	Assert(!(pCtx->ss.Attr.u & 0xf00));
5324	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
5325	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
5326	\|\| !(pCtx->ss.Attr.n.u1Granularity));
5327	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
5328	\|\| (pCtx->ss.Attr.n.u1Granularity));
5329	}
5330	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
5331	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
5332	{
5333	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5334	Assert(pCtx->ds.Attr.n.u1Present);
5335	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
5336	Assert(!(pCtx->ds.Attr.u & 0xf00));
5337	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
5338	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
5339	\|\| !(pCtx->ds.Attr.n.u1Granularity));
5340	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
5341	\|\| (pCtx->ds.Attr.n.u1Granularity));
5342	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5343	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
5344	}
5345	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
5346	{
5347	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5348	Assert(pCtx->es.Attr.n.u1Present);
5349	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
5350	Assert(!(pCtx->es.Attr.u & 0xf00));
5351	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
5352	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
5353	\|\| !(pCtx->es.Attr.n.u1Granularity));
5354	Assert( !(pCtx->es.u32Limit & 0xfff00000)
5355	\|\| (pCtx->es.Attr.n.u1Granularity));
5356	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5357	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
5358	}
5359	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
5360	{
5361	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5362	Assert(pCtx->fs.Attr.n.u1Present);
5363	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
5364	Assert(!(pCtx->fs.Attr.u & 0xf00));
5365	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
5366	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
5367	\|\| !(pCtx->fs.Attr.n.u1Granularity));
5368	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
5369	\|\| (pCtx->fs.Attr.n.u1Granularity));
5370	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5371	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5372	}
5373	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
5374	{
5375	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5376	Assert(pCtx->gs.Attr.n.u1Present);
5377	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
5378	Assert(!(pCtx->gs.Attr.u & 0xf00));
5379	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
5380	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
5381	\|\| !(pCtx->gs.Attr.n.u1Granularity));
5382	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
5383	\|\| (pCtx->gs.Attr.n.u1Granularity));
5384	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5385	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5386	}
5387	/* 64-bit capable CPUs. */
5388	# if HC_ARCH_BITS == 64
5389	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5390	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
5391	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
5392	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
5393	# endif
5394	}
5395	else if ( CPUMIsGuestInV86ModeEx(pCtx)
5396	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
5397	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
5398	{
5399	/* Real and v86 mode checks. */
5400	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
5401	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
5402	if (pVmcsInfo->RealMode.fRealOnV86Active)
5403	{
5404	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
5405	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
5406	}
5407	else
5408	{
5409	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
5410	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
5411	}
5412
5413	/* CS */
5414	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
5415	Assert(pCtx->cs.u32Limit == 0xffff);
5416	Assert(u32CSAttr == 0xf3);
5417	/* SS */
5418	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
5419	Assert(pCtx->ss.u32Limit == 0xffff);
5420	Assert(u32SSAttr == 0xf3);
5421	/* DS */
5422	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
5423	Assert(pCtx->ds.u32Limit == 0xffff);
5424	Assert(u32DSAttr == 0xf3);
5425	/* ES */
5426	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
5427	Assert(pCtx->es.u32Limit == 0xffff);
5428	Assert(u32ESAttr == 0xf3);
5429	/* FS */
5430	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
5431	Assert(pCtx->fs.u32Limit == 0xffff);
5432	Assert(u32FSAttr == 0xf3);
5433	/* GS */
5434	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
5435	Assert(pCtx->gs.u32Limit == 0xffff);
5436	Assert(u32GSAttr == 0xf3);
5437	/* 64-bit capable CPUs. */
5438	# if HC_ARCH_BITS == 64
5439	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5440	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
5441	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
5442	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
5443	# endif
5444	}
5445	}
5446	#endif /* VBOX_STRICT */
5447
5448
5449	/**
5450	* Exports a guest segment register into the guest-state area in the VMCS.
5451	*
5452	* @returns VBox status code.
5453	* @param pVCpu The cross context virtual CPU structure.
5454	* @param pVmcsInfo The VMCS info. object.
5455	* @param iSegReg The segment register number (X86_SREG_XXX).
5456	* @param pSelReg Pointer to the segment selector.
5457	*
5458	* @remarks No-long-jump zone!!!
5459	*/
5460	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
5461	{
5462	Assert(iSegReg < X86_SREG_COUNT);
5463	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
5464	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
5465	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
5466	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
5467
5468	uint32_t u32Access = pSelReg->Attr.u;
5469	if (pVmcsInfo->RealMode.fRealOnV86Active)
5470	{
5471	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
5472	u32Access = 0xf3;
5473	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5474	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5475	RT_NOREF_PV(pVCpu);
5476	}
5477	else
5478	{
5479	/*
5480	* The way to differentiate between whether this is really a null selector or was just
5481	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
5482	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
5483	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
5484	* NULL selectors loaded in protected-mode have their attribute as 0.
5485	*/
5486	if (!u32Access)
5487	u32Access = X86DESCATTR_UNUSABLE;
5488	}
5489
5490	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
5491	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
5492	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
5493
5494	/*
5495	* Commit it to the VMCS.
5496	*/
5497	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
5498	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
5499	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
5500	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
5501	AssertRCReturn(rc, rc);
5502	return rc;
5503	}
5504
5505
5506	/**
5507	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
5508	* area in the VMCS.
5509	*
5510	* @returns VBox status code.
5511	* @param pVCpu The cross context virtual CPU structure.
5512	* @param pVmxTransient The VMX-transient structure.
5513	*
5514	* @remarks Will import guest CR0 on strict builds during validation of
5515	* segments.
5516	* @remarks No-long-jump zone!!!
5517	*/
5518	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5519	{
5520	int rc = VERR_INTERNAL_ERROR_5;
5521	PVM pVM = pVCpu->CTX_SUFF(pVM);
5522	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5523	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5524
5525	/*
5526	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
5527	*/
5528	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
5529	{
5530	#ifdef VBOX_WITH_REM
5531	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
5532	{
5533	Assert(!pVmxTransient->fIsNestedGuest);
5534	Assert(pVM->hm.s.vmx.pRealModeTSS);
5535	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
5536	if ( pVmcsInfo->fWasInRealMode
5537	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
5538	{
5539	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
5540	in real-mode (e.g. OpenBSD 4.0) */
5541	REMFlushTBs(pVM);
5542	Log4Func(("Switch to protected mode detected!\n"));
5543	pVmcsInfo->fWasInRealMode = false;
5544	}
5545	}
5546	#endif
5547	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
5548	{
5549	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
5550	if (pVmcsInfo->RealMode.fRealOnV86Active)
5551	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
5552	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
5553	AssertRCReturn(rc, rc);
5554	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
5555	}
5556
5557	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
5558	{
5559	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
5560	if (pVmcsInfo->RealMode.fRealOnV86Active)
5561	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
5562	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
5563	AssertRCReturn(rc, rc);
5564	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
5565	}
5566
5567	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
5568	{
5569	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
5570	if (pVmcsInfo->RealMode.fRealOnV86Active)
5571	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
5572	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
5573	AssertRCReturn(rc, rc);
5574	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
5575	}
5576
5577	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
5578	{
5579	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
5580	if (pVmcsInfo->RealMode.fRealOnV86Active)
5581	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
5582	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
5583	AssertRCReturn(rc, rc);
5584	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
5585	}
5586
5587	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
5588	{
5589	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
5590	if (pVmcsInfo->RealMode.fRealOnV86Active)
5591	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
5592	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
5593	AssertRCReturn(rc, rc);
5594	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
5595	}
5596
5597	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
5598	{
5599	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
5600	if (pVmcsInfo->RealMode.fRealOnV86Active)
5601	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
5602	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
5603	AssertRCReturn(rc, rc);
5604	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
5605	}
5606
5607	#ifdef VBOX_STRICT
5608	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
5609	#endif
5610	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
5611	pCtx->cs.Attr.u));
5612	}
5613
5614	/*
5615	* Guest TR.
5616	*/
5617	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
5618	{
5619	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
5620
5621	/*
5622	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
5623	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
5624	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
5625	*/
5626	uint16_t u16Sel;
5627	uint32_t u32Limit;
5628	uint64_t u64Base;
5629	uint32_t u32AccessRights;
5630	if (!pVmcsInfo->RealMode.fRealOnV86Active)
5631	{
5632	u16Sel = pCtx->tr.Sel;
5633	u32Limit = pCtx->tr.u32Limit;
5634	u64Base = pCtx->tr.u64Base;
5635	u32AccessRights = pCtx->tr.Attr.u;
5636	}
5637	else
5638	{
5639	Assert(!pVmxTransient->fIsNestedGuest);
5640	Assert(pVM->hm.s.vmx.pRealModeTSS);
5641	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
5642
5643	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
5644	RTGCPHYS GCPhys;
5645	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
5646	AssertRCReturn(rc, rc);
5647
5648	X86DESCATTR DescAttr;
5649	DescAttr.u = 0;
5650	DescAttr.n.u1Present = 1;
5651	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
5652
5653	u16Sel = 0;
5654	u32Limit = HM_VTX_TSS_SIZE;
5655	u64Base = GCPhys;
5656	u32AccessRights = DescAttr.u;
5657	}
5658
5659	/* Validate. */
5660	Assert(!(u16Sel & RT_BIT(2)));
5661	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
5662	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
5663	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
5664	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
5665	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
5666	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
5667	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
5668	Assert( (u32Limit & 0xfff) == 0xfff
5669	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
5670	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
5671	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
5672
5673	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
5674	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
5675	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
5676	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
5677	AssertRCReturn(rc, rc);
5678
5679	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
5680	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
5681	}
5682
5683	/*
5684	* Guest GDTR.
5685	*/
5686	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
5687	{
5688	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
5689
5690	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
5691	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
5692	AssertRCReturn(rc, rc);
5693
5694	/* Validate. */
5695	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5696
5697	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
5698	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
5699	}
5700
5701	/*
5702	* Guest LDTR.
5703	*/
5704	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
5705	{
5706	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
5707
5708	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
5709	uint32_t u32Access;
5710	if ( !pVmxTransient->fIsNestedGuest
5711	&& !pCtx->ldtr.Attr.u)
5712	u32Access = X86DESCATTR_UNUSABLE;
5713	else
5714	u32Access = pCtx->ldtr.Attr.u;
5715
5716	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
5717	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
5718	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
5719	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
5720	AssertRCReturn(rc, rc);
5721
5722	/* Validate. */
5723	if (!(u32Access & X86DESCATTR_UNUSABLE))
5724	{
5725	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
5726	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
5727	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
5728	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
5729	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
5730	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
5731	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
5732	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
5733	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
5734	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
5735	}
5736
5737	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
5738	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
5739	}
5740
5741	/*
5742	* Guest IDTR.
5743	*/
5744	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
5745	{
5746	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
5747
5748	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
5749	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
5750	AssertRCReturn(rc, rc);
5751
5752	/* Validate. */
5753	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5754
5755	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
5756	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
5757	}
5758
5759	return VINF_SUCCESS;
5760	}
5761
5762
5763	/**
5764	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
5765	* areas.
5766	*
5767	* These MSRs will automatically be loaded to the host CPU on every successful
5768	* VM-entry and stored from the host CPU on every successful VM-exit.
5769	*
5770	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
5771	* actual host MSR values are not- updated here for performance reasons. See
5772	* hmR0VmxExportHostMsrs().
5773	*
5774	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
5775	*
5776	* @returns VBox status code.
5777	* @param pVCpu The cross context virtual CPU structure.
5778	* @param pVmxTransient The VMX-transient structure.
5779	*
5780	* @remarks No-long-jump zone!!!
5781	*/
5782	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5783	{
5784	AssertPtr(pVCpu);
5785	AssertPtr(pVmxTransient);
5786
5787	PVM pVM = pVCpu->CTX_SUFF(pVM);
5788	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5789
5790	/*
5791	* MSRs that we use the auto-load/store MSR area in the VMCS.
5792	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
5793	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
5794	*
5795	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
5796	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
5797	* emulation, nothing to do here.
5798	*/
5799	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
5800	{
5801	if ( !pVmxTransient->fIsNestedGuest
5802	&& pVM->hm.s.fAllow64BitGuests)
5803	{
5804	#if HC_ARCH_BITS == 32
5805	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
5806	Assert(!pVmxTransient->fIsNestedGuest);
5807
5808	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
5809	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
5810	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
5811	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
5812	AssertRCReturn(rc, rc);
5813	#endif
5814	}
5815	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
5816	}
5817
5818	/*
5819	* Guest Sysenter MSRs.
5820	*/
5821	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
5822	{
5823	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
5824
5825	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
5826	{
5827	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
5828	AssertRCReturn(rc, rc);
5829	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
5830	}
5831
5832	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
5833	{
5834	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
5835	AssertRCReturn(rc, rc);
5836	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
5837	}
5838
5839	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
5840	{
5841	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
5842	AssertRCReturn(rc, rc);
5843	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
5844	}
5845	}
5846
5847	/*
5848	* Guest/host EFER MSR.
5849	*/
5850	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
5851	{
5852	/* Whether we are using the VMCS to swap the EFER MSR must have been
5853	determined earlier while exporting VM-entry/VM-exit controls. */
5854	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
5855	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
5856
5857	if (hmR0VmxShouldSwapEferMsr(pVCpu))
5858	{
5859	/*
5860	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
5861	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
5862	*/
5863	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
5864	{
5865	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
5866	AssertRCReturn(rc, rc);
5867	}
5868	else
5869	{
5870	/*
5871	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
5872	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
5873	*/
5874	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
5875	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5876	AssertRCReturn(rc, rc);
5877	}
5878	}
5879	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
5880	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
5881
5882	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
5883	}
5884
5885	/*
5886	* Other MSRs.
5887	* Speculation Control (R/W).
5888	*/
5889	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
5890	{
5891	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
5892	if (pVM->cpum.ro.GuestFeatures.fIbrs)
5893	{
5894	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
5895	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5896	AssertRCReturn(rc, rc);
5897	}
5898	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
5899	}
5900
5901	return VINF_SUCCESS;
5902	}
5903
5904
5905	/**
5906	* Selects up the appropriate function to run guest code.
5907	*
5908	* @returns VBox status code.
5909	* @param pVCpu The cross context virtual CPU structure.
5910	* @param pVmxTransient The VMX-transient structure.
5911	*
5912	* @remarks No-long-jump zone!!!
5913	*/
5914	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5915	{
5916	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5917	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5918
5919	if (CPUMIsGuestInLongModeEx(pCtx))
5920	{
5921	#ifndef VBOX_ENABLE_64_BITS_GUESTS
5922	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5923	#endif
5924	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
5925	#if HC_ARCH_BITS == 32
5926	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
5927	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
5928	{
5929	#ifdef VBOX_STRICT
5930	if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5931	{
5932	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5933	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5934	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5935	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5936	("fCtxChanged=%#RX64\n", fCtxChanged));
5937	}
5938	#endif
5939	pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
5940
5941	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
5942	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
5943	pVmcsInfo->fSwitchedTo64on32 = true;
5944	Log4Func(("Selected 64-bit switcher\n"));
5945	}
5946	#else
5947	/* 64-bit host. */
5948	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
5949	#endif
5950	}
5951	else
5952	{
5953	/* Guest is not in long mode, use the 32-bit handler. */
5954	#if HC_ARCH_BITS == 32
5955	if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
5956	&& !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
5957	&& pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5958	{
5959	# ifdef VBOX_STRICT
5960	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5961	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5962	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5963	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5964	("fCtxChanged=%#RX64\n", fCtxChanged));
5965	# endif
5966	}
5967	# ifdef VBOX_ENABLE_64_BITS_GUESTS
5968	/*
5969	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
5970	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
5971	* switcher flag now because we know the guest is in a sane state where it's safe
5972	* to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
5973	* the much faster 32-bit switcher again.
5974	*/
5975	if (!pVmcsInfo->fSwitchedTo64on32)
5976	{
5977	if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
5978	Log4Func(("Selected 32-bit switcher\n"));
5979	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5980	}
5981	else
5982	{
5983	Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
5984	if ( pVmcsInfo->RealMode.fRealOnV86Active
5985	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
5986	{
5987	pVmcsInfo->fSwitchedTo64on32 = false;
5988	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5989	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
5990	\| HM_CHANGED_VMX_ENTRY_EXIT_CTLS
5991	\| HM_CHANGED_HOST_CONTEXT);
5992	Log4Func(("Selected 32-bit switcher (safe)\n"));
5993	}
5994	}
5995	# else
5996	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5997	# endif
5998	#else
5999	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6000	#endif
6001	}
6002	Assert(pVmcsInfo->pfnStartVM);
6003	return VINF_SUCCESS;
6004	}
6005
6006
6007	/**
6008	* Wrapper for running the guest code in VT-x.
6009	*
6010	* @returns VBox status code, no informational status codes.
6011	* @param pVCpu The cross context virtual CPU structure.
6012	* @param pVmxTransient The VMX-transient structure.
6013	*
6014	* @remarks No-long-jump zone!!!
6015	*/
6016	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6017	{
6018	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6019	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6020	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6021
6022	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6023
6024	/*
6025	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6026	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6027	* callee-saved and thus the need for this XMM wrapper.
6028	*
6029	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6030	*/
6031	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6032	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6033	PVM pVM = pVCpu->CTX_SUFF(pVM);
6034	#ifdef VBOX_WITH_KERNEL_USING_XMM
6035	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6036	#else
6037	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
6038	#endif
6039	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6040	return rc;
6041	}
6042
6043
6044	/**
6045	* Reports world-switch error and dumps some useful debug info.
6046	*
6047	* @param pVCpu The cross context virtual CPU structure.
6048	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6049	* @param pVmxTransient The VMX-transient structure (only
6050	* exitReason updated).
6051	*/
6052	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6053	{
6054	Assert(pVCpu);
6055	Assert(pVmxTransient);
6056	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6057
6058	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6059	switch (rcVMRun)
6060	{
6061	case VERR_VMX_INVALID_VMXON_PTR:
6062	AssertFailed();
6063	break;
6064	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6065	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6066	{
6067	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6068	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6069	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
6070	AssertRC(rc);
6071
6072	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6073	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6074	Cannot do it here as we may have been long preempted. */
6075
6076	#ifdef VBOX_STRICT
6077	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6078	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6079	pVmxTransient->uExitReason));
6080	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6081	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6082	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6083	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6084	else
6085	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6086	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6087	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6088
6089	/* VMX control bits. */
6090	uint32_t u32Val;
6091	uint64_t u64Val;
6092	RTHCUINTREG uHCReg;
6093	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
6094	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
6095	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
6096	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
6097	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
6098	{
6099	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
6100	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
6101	}
6102	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
6103	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
6104	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
6105	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
6106	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
6107	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
6108	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
6109	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
6110	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
6111	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
6112	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
6113	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
6114	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
6115	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
6116	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
6117	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
6118	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6119	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
6120	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6121	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
6122	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
6123	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
6124	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
6125	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
6126	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
6127	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
6128	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
6129	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
6130	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
6131	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6132	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
6133	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
6134	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
6135	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6136	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6137	{
6138	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
6139	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
6140	}
6141
6142	/* Guest bits. */
6143	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
6144	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
6145	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
6146	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
6147	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
6148	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
6149	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
6150	{
6151	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
6152	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
6153	}
6154
6155	/* Host bits. */
6156	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
6157	Log4(("Host CR0 %#RHr\n", uHCReg));
6158	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
6159	Log4(("Host CR3 %#RHr\n", uHCReg));
6160	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
6161	Log4(("Host CR4 %#RHr\n", uHCReg));
6162
6163	RTGDTR HostGdtr;
6164	PCX86DESCHC pDesc;
6165	ASMGetGDTR(&HostGdtr);
6166	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
6167	Log4(("Host CS %#08x\n", u32Val));
6168	if (u32Val < HostGdtr.cbGdt)
6169	{
6170	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6171	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
6172	}
6173
6174	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
6175	Log4(("Host DS %#08x\n", u32Val));
6176	if (u32Val < HostGdtr.cbGdt)
6177	{
6178	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6179	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
6180	}
6181
6182	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
6183	Log4(("Host ES %#08x\n", u32Val));
6184	if (u32Val < HostGdtr.cbGdt)
6185	{
6186	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6187	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
6188	}
6189
6190	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
6191	Log4(("Host FS %#08x\n", u32Val));
6192	if (u32Val < HostGdtr.cbGdt)
6193	{
6194	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6195	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
6196	}
6197
6198	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
6199	Log4(("Host GS %#08x\n", u32Val));
6200	if (u32Val < HostGdtr.cbGdt)
6201	{
6202	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6203	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
6204	}
6205
6206	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
6207	Log4(("Host SS %#08x\n", u32Val));
6208	if (u32Val < HostGdtr.cbGdt)
6209	{
6210	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6211	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
6212	}
6213
6214	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
6215	Log4(("Host TR %#08x\n", u32Val));
6216	if (u32Val < HostGdtr.cbGdt)
6217	{
6218	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6219	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
6220	}
6221
6222	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
6223	Log4(("Host TR Base %#RHv\n", uHCReg));
6224	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
6225	Log4(("Host GDTR Base %#RHv\n", uHCReg));
6226	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
6227	Log4(("Host IDTR Base %#RHv\n", uHCReg));
6228	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
6229	Log4(("Host SYSENTER CS %#08x\n", u32Val));
6230	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
6231	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
6232	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
6233	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
6234	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
6235	Log4(("Host RSP %#RHv\n", uHCReg));
6236	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
6237	Log4(("Host RIP %#RHv\n", uHCReg));
6238	# if HC_ARCH_BITS == 64
6239	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6240	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6241	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6242	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6243	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6244	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6245	# endif
6246	#endif /* VBOX_STRICT */
6247	break;
6248	}
6249
6250	default:
6251	/* Impossible */
6252	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6253	break;
6254	}
6255	}
6256
6257
6258	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
6259	# ifndef VMX_USE_CACHED_VMCS_ACCESSES
6260	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
6261	# endif
6262
6263	/**
6264	* Initialize the VMCS-Read cache.
6265	*
6266	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
6267	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
6268	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
6269	* (those that have a 32-bit FULL & HIGH part).
6270	*
6271	* @param pVCpu The cross context virtual CPU structure.
6272	*/
6273	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
6274	{
6275	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
6276	do { \
6277	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
6278	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
6279	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
6280	++cReadFields; \
6281	} while (0)
6282
6283	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6284	uint32_t cReadFields = 0;
6285
6286	/*
6287	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
6288	* and serve to indicate exceptions to the rules.
6289	*/
6290
6291	/* Guest-natural selector base fields. */
6292	#if 0
6293	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
6294	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
6295	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
6296	#endif
6297	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
6298	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
6299	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
6300	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
6301	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
6302	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
6303	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
6304	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
6305	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
6306	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
6307	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
6308	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
6309	#if 0
6310	/* Unused natural width guest-state fields. */
6311	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
6312	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
6313	#endif
6314	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
6315	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
6316
6317	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
6318	these 64-bit fields (using "FULL" and "HIGH" fields). */
6319	#if 0
6320	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
6321	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
6322	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
6323	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
6324	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
6325	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
6326	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
6327	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
6328	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
6329	#endif
6330
6331	/* Natural width guest-state fields. */
6332	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
6333	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
6334
6335	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6336	{
6337	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
6338	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
6339	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
6340	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
6341	}
6342	else
6343	{
6344	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
6345	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
6346	}
6347
6348	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
6349	}
6350
6351
6352	/**
6353	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
6354	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
6355	* darwin, running 64-bit guests).
6356	*
6357	* @returns VBox status code.
6358	* @param pVCpu The cross context virtual CPU structure.
6359	* @param idxField The VMCS field encoding.
6360	* @param u64Val 16, 32 or 64-bit value.
6361	*/
6362	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6363	{
6364	int rc;
6365	switch (idxField)
6366	{
6367	/*
6368	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
6369	*/
6370	/* 64-bit Control fields. */
6371	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
6372	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
6373	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
6374	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
6375	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
6376	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
6377	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
6378	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
6379	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
6380	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
6381	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
6382	case VMX_VMCS64_CTRL_EPTP_FULL:
6383	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
6384	/* 64-bit Guest-state fields. */
6385	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
6386	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
6387	case VMX_VMCS64_GUEST_PAT_FULL:
6388	case VMX_VMCS64_GUEST_EFER_FULL:
6389	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
6390	case VMX_VMCS64_GUEST_PDPTE0_FULL:
6391	case VMX_VMCS64_GUEST_PDPTE1_FULL:
6392	case VMX_VMCS64_GUEST_PDPTE2_FULL:
6393	case VMX_VMCS64_GUEST_PDPTE3_FULL:
6394	/* 64-bit Host-state fields. */
6395	case VMX_VMCS64_HOST_PAT_FULL:
6396	case VMX_VMCS64_HOST_EFER_FULL:
6397	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
6398	{
6399	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6400	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
6401	break;
6402	}
6403
6404	/*
6405	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
6406	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
6407	*/
6408	/* Natural-width Guest-state fields. */
6409	case VMX_VMCS_GUEST_CR3:
6410	case VMX_VMCS_GUEST_ES_BASE:
6411	case VMX_VMCS_GUEST_CS_BASE:
6412	case VMX_VMCS_GUEST_SS_BASE:
6413	case VMX_VMCS_GUEST_DS_BASE:
6414	case VMX_VMCS_GUEST_FS_BASE:
6415	case VMX_VMCS_GUEST_GS_BASE:
6416	case VMX_VMCS_GUEST_LDTR_BASE:
6417	case VMX_VMCS_GUEST_TR_BASE:
6418	case VMX_VMCS_GUEST_GDTR_BASE:
6419	case VMX_VMCS_GUEST_IDTR_BASE:
6420	case VMX_VMCS_GUEST_RSP:
6421	case VMX_VMCS_GUEST_RIP:
6422	case VMX_VMCS_GUEST_SYSENTER_ESP:
6423	case VMX_VMCS_GUEST_SYSENTER_EIP:
6424	{
6425	if (!(RT_HI_U32(u64Val)))
6426	{
6427	/* If this field is 64-bit, VT-x will zero out the top bits. */
6428	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6429	}
6430	else
6431	{
6432	/* Assert that only the 32->64 switcher case should ever come here. */
6433	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
6434	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
6435	}
6436	break;
6437	}
6438
6439	default:
6440	{
6441	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
6442	rc = VERR_INVALID_PARAMETER;
6443	break;
6444	}
6445	}
6446	AssertRCReturn(rc, rc);
6447	return rc;
6448	}
6449
6450
6451	/**
6452	* Queue up a VMWRITE by using the VMCS write cache.
6453	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
6454	*
6455	* @param pVCpu The cross context virtual CPU structure.
6456	* @param idxField The VMCS field encoding.
6457	* @param u64Val 16, 32 or 64-bit value.
6458	*/
6459	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6460	{
6461	AssertPtr(pVCpu);
6462	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6463
6464	AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
6465	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
6466
6467	/* Make sure there are no duplicates. */
6468	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
6469	{
6470	if (pCache->Write.aField[i] == idxField)
6471	{
6472	pCache->Write.aFieldVal[i] = u64Val;
6473	return VINF_SUCCESS;
6474	}
6475	}
6476
6477	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
6478	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
6479	pCache->Write.cValidEntries++;
6480	return VINF_SUCCESS;
6481	}
6482	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
6483
6484
6485	/**
6486	* Sets up the usage of TSC-offsetting and updates the VMCS.
6487	*
6488	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6489	* VMX-preemption timer.
6490	*
6491	* @returns VBox status code.
6492	* @param pVCpu The cross context virtual CPU structure.
6493	* @param pVmxTransient The VMX-transient structure.
6494	*
6495	* @remarks No-long-jump zone!!!
6496	*/
6497	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6498	{
6499	bool fOffsettedTsc;
6500	bool fParavirtTsc;
6501	uint64_t uTscOffset;
6502	PVM pVM = pVCpu->CTX_SUFF(pVM);
6503	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);;
6504
6505	if (pVM->hm.s.vmx.fUsePreemptTimer)
6506	{
6507	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6508
6509	/* Make sure the returned values have sane upper and lower boundaries. */
6510	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6511	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6512	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6513	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6514
6515	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6516	* preemption timers here. We probably need to clamp the preemption timer,
6517	* after converting the timer value to the host. */
6518	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6519	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6520	AssertRC(rc);
6521	}
6522	else
6523	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6524
6525	if (fParavirtTsc)
6526	{
6527	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6528	information before every VM-entry, hence disable it for performance sake. */
6529	#if 0
6530	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
6531	AssertRC(rc);
6532	#endif
6533	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
6534	}
6535
6536	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
6537	if ( fOffsettedTsc
6538	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
6539	{
6540	if (pVmxTransient->fIsNestedGuest)
6541	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
6542	if (pVmcsInfo->u64TscOffset != uTscOffset)
6543	{
6544	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
6545	AssertRC(rc);
6546	pVmcsInfo->u64TscOffset = uTscOffset;
6547	}
6548
6549	if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
6550	{
6551	uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
6552	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6553	AssertRC(rc);
6554	pVmcsInfo->u32ProcCtls = uProcCtls;
6555	}
6556	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
6557	}
6558	else
6559	{
6560	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
6561	if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
6562	{
6563	uProcCtls \|= VMX_PROC_CTLS_RDTSC_EXIT;
6564	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6565	AssertRC(rc);
6566	pVmcsInfo->u32ProcCtls = uProcCtls;
6567	}
6568	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
6569	}
6570	}
6571
6572
6573	/**
6574	* Gets the IEM exception flags for the specified vector and IDT vectoring /
6575	* VM-exit interruption info type.
6576	*
6577	* @returns The IEM exception flags.
6578	* @param uVector The event vector.
6579	* @param uVmxEventType The VMX event type.
6580	*
6581	* @remarks This function currently only constructs flags required for
6582	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
6583	* and CR2 aspects of an exception are not included).
6584	*/
6585	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
6586	{
6587	uint32_t fIemXcptFlags;
6588	switch (uVmxEventType)
6589	{
6590	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6591	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6592	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6593	break;
6594
6595	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6596	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6597	break;
6598
6599	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6600	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
6601	break;
6602
6603	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
6604	{
6605	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6606	if (uVector == X86_XCPT_BP)
6607	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
6608	else if (uVector == X86_XCPT_OF)
6609	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
6610	else
6611	{
6612	fIemXcptFlags = 0;
6613	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
6614	}
6615	break;
6616	}
6617
6618	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6619	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6620	break;
6621
6622	default:
6623	fIemXcptFlags = 0;
6624	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
6625	break;
6626	}
6627	return fIemXcptFlags;
6628	}
6629
6630
6631	/**
6632	* Sets an event as a pending event to be injected into the guest.
6633	*
6634	* @param pVCpu The cross context virtual CPU structure.
6635	* @param u32IntInfo The VM-entry interruption-information field.
6636	* @param cbInstr The VM-entry instruction length in bytes (for software
6637	* interrupts, exceptions and privileged software
6638	* exceptions).
6639	* @param u32ErrCode The VM-entry exception error code.
6640	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
6641	* page-fault.
6642	*/
6643	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
6644	RTGCUINTPTR GCPtrFaultAddress)
6645	{
6646	Assert(!pVCpu->hm.s.Event.fPending);
6647	pVCpu->hm.s.Event.fPending = true;
6648	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
6649	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
6650	pVCpu->hm.s.Event.cbInstr = cbInstr;
6651	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
6652	}
6653
6654
6655	/**
6656	* Sets an external interrupt as pending-for-injection into the VM.
6657	*
6658	* @param pVCpu The cross context virtual CPU structure.
6659	* @param u8Interrupt The external interrupt vector.
6660	*/
6661	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
6662	{
6663	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
6664	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6665	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6666	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6667	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6668	}
6669
6670
6671	/**
6672	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
6673	*
6674	* @param pVCpu The cross context virtual CPU structure.
6675	*/
6676	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
6677	{
6678	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
6679	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
6680	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6681	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6682	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6683	}
6684
6685
6686	/**
6687	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
6688	*
6689	* @param pVCpu The cross context virtual CPU structure.
6690	*/
6691	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
6692	{
6693	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
6694	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6695	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6696	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6697	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6698	}
6699
6700
6701	/**
6702	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
6703	*
6704	* @param pVCpu The cross context virtual CPU structure.
6705	*/
6706	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
6707	{
6708	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
6709	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6710	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6711	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6712	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6713	}
6714
6715
6716	/**
6717	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
6718	*
6719	* @param pVCpu The cross context virtual CPU structure.
6720	*/
6721	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
6722	{
6723	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
6724	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6725	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6726	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6727	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6728	}
6729
6730
6731	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6732	/**
6733	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
6734	*
6735	* @param pVCpu The cross context virtual CPU structure.
6736	* @param u32ErrCode The error code for the general-protection exception.
6737	*/
6738	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
6739	{
6740	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
6741	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6742	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6743	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6744	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6745	}
6746
6747
6748	/**
6749	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
6750	*
6751	* @param pVCpu The cross context virtual CPU structure.
6752	* @param u32ErrCode The error code for the stack exception.
6753	*/
6754	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
6755	{
6756	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
6757	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6758	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6759	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6760	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6761	}
6762
6763
6764	/**
6765	* Decodes the memory operand of an instruction that caused a VM-exit.
6766	*
6767	* The VM-exit qualification field provides the displacement field for memory
6768	* operand instructions, if any.
6769	*
6770	* @returns Strict VBox status code (i.e. informational status codes too).
6771	* @retval VINF_SUCCESS if the operand was successfully decoded.
6772	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
6773	* operand.
6774	* @param pVCpu The cross context virtual CPU structure.
6775	* @param uExitInstrInfo The VM-exit instruction information field.
6776	* @param enmMemAccess The memory operand's access type (read or write).
6777	* @param GCPtrDisp The instruction displacement field, if any. For
6778	* RIP-relative addressing pass RIP + displacement here.
6779	* @param pGCPtrMem Where to store the effective destination memory address.
6780	*/
6781	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
6782	PRTGCPTR pGCPtrMem)
6783	{
6784	Assert(pGCPtrMem);
6785	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
6786	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
6787	\| CPUMCTX_EXTRN_CR0);
6788
6789	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
6790	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
6791	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
6792
6793	VMXEXITINSTRINFO ExitInstrInfo;
6794	ExitInstrInfo.u = uExitInstrInfo;
6795	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
6796	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
6797	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
6798	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
6799	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
6800	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
6801	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
6802	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
6803	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
6804
6805	/*
6806	* Validate instruction information.
6807	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
6808	*/
6809	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
6810	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
6811	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
6812	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
6813	AssertLogRelMsgReturn(fIsMemOperand,
6814	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
6815
6816	/*
6817	* Compute the complete effective address.
6818	*
6819	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
6820	* See AMD spec. 4.5.2 "Segment Registers".
6821	*/
6822	RTGCPTR GCPtrMem = GCPtrDisp;
6823	if (fBaseRegValid)
6824	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
6825	if (fIdxRegValid)
6826	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
6827
6828	RTGCPTR const GCPtrOff = GCPtrMem;
6829	if ( !fIsLongMode
6830	\|\| iSegReg >= X86_SREG_FS)
6831	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
6832	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
6833
6834	/*
6835	* Validate effective address.
6836	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
6837	*/
6838	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
6839	Assert(cbAccess > 0);
6840	if (fIsLongMode)
6841	{
6842	if (X86_IS_CANONICAL(GCPtrMem))
6843	{
6844	*pGCPtrMem = GCPtrMem;
6845	return VINF_SUCCESS;
6846	}
6847
6848	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
6849	* "Data Limit Checks in 64-bit Mode". */
6850	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
6851	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6852	return VINF_HM_PENDING_XCPT;
6853	}
6854
6855	/*
6856	* This is a watered down version of iemMemApplySegment().
6857	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
6858	* and segment CPL/DPL checks are skipped.
6859	*/
6860	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
6861	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
6862	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
6863
6864	/* Check if the segment is present and usable. */
6865	if ( pSel->Attr.n.u1Present
6866	&& !pSel->Attr.n.u1Unusable)
6867	{
6868	Assert(pSel->Attr.n.u1DescType);
6869	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
6870	{
6871	/* Check permissions for the data segment. */
6872	if ( enmMemAccess == VMXMEMACCESS_WRITE
6873	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
6874	{
6875	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6876	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
6877	return VINF_HM_PENDING_XCPT;
6878	}
6879
6880	/* Check limits if it's a normal data segment. */
6881	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
6882	{
6883	if ( GCPtrFirst32 > pSel->u32Limit
6884	\|\| GCPtrLast32 > pSel->u32Limit)
6885	{
6886	Log4Func(("Data segment limit exceeded."
6887	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6888	GCPtrLast32, pSel->u32Limit));
6889	if (iSegReg == X86_SREG_SS)
6890	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6891	else
6892	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6893	return VINF_HM_PENDING_XCPT;
6894	}
6895	}
6896	else
6897	{
6898	/* Check limits if it's an expand-down data segment.
6899	Note! The upper boundary is defined by the B bit, not the G bit! */
6900	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
6901	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
6902	{
6903	Log4Func(("Expand-down data segment limit exceeded."
6904	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6905	GCPtrLast32, pSel->u32Limit));
6906	if (iSegReg == X86_SREG_SS)
6907	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6908	else
6909	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6910	return VINF_HM_PENDING_XCPT;
6911	}
6912	}
6913	}
6914	else
6915	{
6916	/* Check permissions for the code segment. */
6917	if ( enmMemAccess == VMXMEMACCESS_WRITE
6918	\|\| ( enmMemAccess == VMXMEMACCESS_READ
6919	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
6920	{
6921	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6922	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6923	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6924	return VINF_HM_PENDING_XCPT;
6925	}
6926
6927	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6928	if ( GCPtrFirst32 > pSel->u32Limit
6929	\|\| GCPtrLast32 > pSel->u32Limit)
6930	{
6931	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6932	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6933	if (iSegReg == X86_SREG_SS)
6934	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6935	else
6936	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6937	return VINF_HM_PENDING_XCPT;
6938	}
6939	}
6940	}
6941	else
6942	{
6943	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6944	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6945	return VINF_HM_PENDING_XCPT;
6946	}
6947
6948	*pGCPtrMem = GCPtrMem;
6949	return VINF_SUCCESS;
6950	}
6951
6952
6953	/**
6954	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6955	* guest attempting to execute a VMX instruction.
6956	*
6957	* @returns Strict VBox status code (i.e. informational status codes too).
6958	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
6959	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
6960	*
6961	* @param pVCpu The cross context virtual CPU structure.
6962	* @param uExitReason The VM-exit reason.
6963	*
6964	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
6965	* @remarks No-long-jump zone!!!
6966	*/
6967	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
6968	{
6969	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
6970	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
6971
6972	if ( CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx)
6973	\|\| ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6974	&& !CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)))
6975	{
6976	Log4Func(("In real/v86-mode or long-mode outside 64-bit code segment -> #UD\n"));
6977	hmR0VmxSetPendingXcptUD(pVCpu);
6978	return VINF_HM_PENDING_XCPT;
6979	}
6980
6981	if (uExitReason == VMX_EXIT_VMXON)
6982	{
6983	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
6984
6985	/*
6986	* We check CR4.VMXE because it is required to be always set while in VMX operation
6987	* by physical CPUs and our CR4 read shadow is only consulted when executing specific
6988	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
6989	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
6990	*/
6991	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
6992	{
6993	Log4Func(("CR4.VMXE is not set -> #UD\n"));
6994	hmR0VmxSetPendingXcptUD(pVCpu);
6995	return VINF_HM_PENDING_XCPT;
6996	}
6997	}
6998	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
6999	{
7000	/*
7001	* The guest has not entered VMX operation but attempted to execute a VMX instruction
7002	* (other than VMXON), we need to raise a #UD.
7003	*/
7004	Log4Func(("Not in VMX root mode -> #UD\n"));
7005	hmR0VmxSetPendingXcptUD(pVCpu);
7006	return VINF_HM_PENDING_XCPT;
7007	}
7008
7009	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
7010	{
7011	/*
7012	* The nested-guest attempted to execute a VMX instruction, cause a VM-exit and let
7013	* the guest hypervisor deal with it.
7014	*/
7015	/** @todo NSTVMX: Trigger a VM-exit */
7016	}
7017
7018	/*
7019	* VMX instructions require CPL 0 except in VMX non-root mode where the VM-exit intercept
7020	* (above) takes preceedence over the CPL check.
7021	*/
7022	if (CPUMGetGuestCPL(pVCpu) > 0)
7023	{
7024	Log4Func(("CPL > 0 -> #GP(0)\n"));
7025	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7026	return VINF_HM_PENDING_XCPT;
7027	}
7028
7029	return VINF_SUCCESS;
7030	}
7031	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7032
7033
7034	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7035	{
7036	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7037
7038	/*
7039	* If VT-x marks the segment as unusable, most other bits remain undefined:
7040	* - For CS the L, D and G bits have meaning.
7041	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7042	* - For the remaining data segments no bits are defined.
7043	*
7044	* The present bit and the unusable bit has been observed to be set at the
7045	* same time (the selector was supposed to be invalid as we started executing
7046	* a V8086 interrupt in ring-0).
7047	*
7048	* What should be important for the rest of the VBox code, is that the P bit is
7049	* cleared. Some of the other VBox code recognizes the unusable bit, but
7050	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7051	* safe side here, we'll strip off P and other bits we don't care about. If
7052	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7053	*
7054	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7055	*/
7056	#ifdef VBOX_STRICT
7057	uint32_t const uAttr = pSelReg->Attr.u;
7058	#endif
7059
7060	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7061	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7062	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7063
7064	#ifdef VBOX_STRICT
7065	VMMRZCallRing3Disable(pVCpu);
7066	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7067	# ifdef DEBUG_bird
7068	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7069	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7070	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7071	# endif
7072	VMMRZCallRing3Enable(pVCpu);
7073	#endif
7074	RT_NOREF2(pVCpu, idxSel);
7075	}
7076
7077
7078	/**
7079	* Imports a guest segment register from the current VMCS into the guest-CPU
7080	* context.
7081	*
7082	* @returns VBox status code.
7083	* @param pVCpu The cross context virtual CPU structure.
7084	* @param iSegReg The segment register number (X86_SREG_XXX).
7085	*
7086	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7087	* do not log!
7088	*/
7089	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7090	{
7091	Assert(iSegReg < X86_SREG_COUNT);
7092
7093	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7094	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7095	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7096	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
7097	uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
7098	#else
7099	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7100	#endif
7101	uint64_t u64Base;
7102	uint32_t u32Sel, u32Limit, u32Attr;
7103	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7104	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7105	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7106	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
7107	if (RT_SUCCESS(rc))
7108	{
7109	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7110	pSelReg->Sel = u32Sel;
7111	pSelReg->ValidSel = u32Sel;
7112	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7113	pSelReg->u32Limit = u32Limit;
7114	pSelReg->u64Base = u64Base;
7115	pSelReg->Attr.u = u32Attr;
7116	if (u32Attr & X86DESCATTR_UNUSABLE)
7117	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7118	}
7119	return rc;
7120	}
7121
7122
7123	/**
7124	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7125	*
7126	* @returns VBox status code.
7127	* @param pVCpu The cross context virtual CPU structure.
7128	*
7129	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7130	* do not log!
7131	*/
7132	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7133	{
7134	uint64_t u64Base;
7135	uint32_t u32Sel, u32Limit, u32Attr;
7136	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7137	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7138	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7139	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7140
7141	if (RT_SUCCESS(rc))
7142	{
7143	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7144	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7145	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7146	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7147	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7148	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7149	if (u32Attr & X86DESCATTR_UNUSABLE)
7150	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7151	}
7152	return rc;
7153	}
7154
7155
7156	/**
7157	* Imports the guest TR from the current VMCS into the guest-CPU context.
7158	*
7159	* @returns VBox status code.
7160	* @param pVCpu The cross context virtual CPU structure.
7161	*
7162	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7163	* do not log!
7164	*/
7165	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7166	{
7167	uint32_t u32Sel, u32Limit, u32Attr;
7168	uint64_t u64Base;
7169	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7170	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7171	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7172	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7173	AssertRCReturn(rc, rc);
7174
7175	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7176	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7177	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7178	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7179	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7180	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7181	/* TR is the only selector that can never be unusable. */
7182	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7183	return VINF_SUCCESS;
7184	}
7185
7186
7187	/**
7188	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7189	*
7190	* @returns VBox status code.
7191	* @param pVCpu The cross context virtual CPU structure.
7192	*
7193	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7194	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7195	* instead!!!
7196	*/
7197	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7198	{
7199	uint64_t u64Val;
7200	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7201	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7202	{
7203	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7204	if (RT_SUCCESS(rc))
7205	{
7206	pCtx->rip = u64Val;
7207	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7208	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7209	}
7210	return rc;
7211	}
7212	return VINF_SUCCESS;
7213	}
7214
7215
7216	/**
7217	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7218	*
7219	* @returns VBox status code.
7220	* @param pVCpu The cross context virtual CPU structure.
7221	* @param pVmcsInfo The VMCS info. object.
7222	*
7223	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7224	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7225	* instead!!!
7226	*/
7227	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7228	{
7229	uint32_t u32Val;
7230	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7231	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7232	{
7233	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7234	if (RT_SUCCESS(rc))
7235	{
7236	pCtx->eflags.u32 = u32Val;
7237
7238	/* Restore eflags for real-on-v86-mode hack. */
7239	if (pVmcsInfo->RealMode.fRealOnV86Active)
7240	{
7241	pCtx->eflags.Bits.u1VM = 0;
7242	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7243	}
7244	}
7245	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7246	return rc;
7247	}
7248	return VINF_SUCCESS;
7249	}
7250
7251
7252	/**
7253	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7254	* context.
7255	*
7256	* @returns VBox status code.
7257	* @param pVCpu The cross context virtual CPU structure.
7258	* @param pVmcsInfo The VMCS info. object.
7259	*
7260	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7261	* do not log!
7262	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7263	* instead!!!
7264	*/
7265	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7266	{
7267	uint32_t u32Val;
7268	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7269	if (RT_SUCCESS(rc))
7270	{
7271	if (!u32Val)
7272	{
7273	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7274	{
7275	rc = hmR0VmxImportGuestRip(pVCpu);
7276	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7277	AssertRCReturn(rc, rc);
7278	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7279	}
7280
7281	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7282	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7283	}
7284	else
7285	{
7286	/*
7287	* We must import RIP here to set our EM interrupt-inhibited state.
7288	* We also import RFLAGS as our code that evaluates pending interrupts
7289	* before VM-entry requires it.
7290	*/
7291	rc = hmR0VmxImportGuestRip(pVCpu);
7292	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7293	if (RT_SUCCESS(rc))
7294	{
7295	if (u32Val & ( VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
7296	\| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7297	{
7298	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7299	}
7300	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7301	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7302
7303	if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
7304	{
7305	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7306	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
7307	}
7308	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7309	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7310	}
7311	}
7312	}
7313	return rc;
7314	}
7315
7316
7317	/**
7318	* Worker for VMXR0ImportStateOnDemand.
7319	*
7320	* @returns VBox status code.
7321	* @param pVCpu The cross context virtual CPU structure.
7322	* @param pVmcsInfo The VMCS info. object.
7323	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7324	*/
7325	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7326	{
7327	#define VMXLOCAL_BREAK_RC(a_rc) \
7328	if (RT_SUCCESS(a_rc)) \
7329	{ } \
7330	else \
7331	break
7332
7333	int rc = VINF_SUCCESS;
7334	PVM pVM = pVCpu->CTX_SUFF(pVM);
7335	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7336	uint64_t u64Val;
7337	uint32_t u32Val;
7338
7339	STAM_PROFILE_ADV_START(& pVCpu->hm.s.StatImportGuestState, x);
7340
7341	/*
7342	* We disable interrupts to make the updating of the state and in particular
7343	* the fExtrn modification atomic wrt to preemption hooks.
7344	*/
7345	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7346
7347	fWhat &= pCtx->fExtrn;
7348	if (fWhat)
7349	{
7350	do
7351	{
7352	if (fWhat & CPUMCTX_EXTRN_RIP)
7353	{
7354	rc = hmR0VmxImportGuestRip(pVCpu);
7355	VMXLOCAL_BREAK_RC(rc);
7356	}
7357
7358	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7359	{
7360	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7361	VMXLOCAL_BREAK_RC(rc);
7362	}
7363
7364	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7365	{
7366	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7367	VMXLOCAL_BREAK_RC(rc);
7368	}
7369
7370	if (fWhat & CPUMCTX_EXTRN_RSP)
7371	{
7372	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
7373	VMXLOCAL_BREAK_RC(rc);
7374	pCtx->rsp = u64Val;
7375	}
7376
7377	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7378	{
7379	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7380	if (fWhat & CPUMCTX_EXTRN_CS)
7381	{
7382	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7383	rc \|= hmR0VmxImportGuestRip(pVCpu);
7384	if (fRealOnV86Active)
7385	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7386	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7387	}
7388	if (fWhat & CPUMCTX_EXTRN_SS)
7389	{
7390	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7391	if (fRealOnV86Active)
7392	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7393	}
7394	if (fWhat & CPUMCTX_EXTRN_DS)
7395	{
7396	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7397	if (fRealOnV86Active)
7398	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7399	}
7400	if (fWhat & CPUMCTX_EXTRN_ES)
7401	{
7402	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7403	if (fRealOnV86Active)
7404	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7405	}
7406	if (fWhat & CPUMCTX_EXTRN_FS)
7407	{
7408	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7409	if (fRealOnV86Active)
7410	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7411	}
7412	if (fWhat & CPUMCTX_EXTRN_GS)
7413	{
7414	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7415	if (fRealOnV86Active)
7416	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7417	}
7418	VMXLOCAL_BREAK_RC(rc);
7419	}
7420
7421	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7422	{
7423	if (fWhat & CPUMCTX_EXTRN_LDTR)
7424	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
7425
7426	if (fWhat & CPUMCTX_EXTRN_GDTR)
7427	{
7428	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
7429	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
7430	pCtx->gdtr.pGdt = u64Val;
7431	pCtx->gdtr.cbGdt = u32Val;
7432	}
7433
7434	/* Guest IDTR. */
7435	if (fWhat & CPUMCTX_EXTRN_IDTR)
7436	{
7437	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
7438	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
7439	pCtx->idtr.pIdt = u64Val;
7440	pCtx->idtr.cbIdt = u32Val;
7441	}
7442
7443	/* Guest TR. */
7444	if (fWhat & CPUMCTX_EXTRN_TR)
7445	{
7446	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7447	don't need to import that one. */
7448	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7449	rc \|= hmR0VmxImportGuestTr(pVCpu);
7450	}
7451	VMXLOCAL_BREAK_RC(rc);
7452	}
7453
7454	if (fWhat & CPUMCTX_EXTRN_DR7)
7455	{
7456	if (!pVCpu->hm.s.fUsingHyperDR7)
7457	{
7458	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
7459	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
7460	VMXLOCAL_BREAK_RC(rc);
7461	pCtx->dr[7] = u32Val;
7462	}
7463	}
7464
7465	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7466	{
7467	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
7468	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
7469	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
7470	pCtx->SysEnter.cs = u32Val;
7471	VMXLOCAL_BREAK_RC(rc);
7472	}
7473
7474	#if HC_ARCH_BITS == 64
7475	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7476	{
7477	if ( pVM->hm.s.fAllow64BitGuests
7478	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7479	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7480	}
7481
7482	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7483	{
7484	if ( pVM->hm.s.fAllow64BitGuests
7485	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7486	{
7487	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7488	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7489	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7490	}
7491	}
7492	#endif
7493
7494	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7495	#if HC_ARCH_BITS == 32
7496	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
7497	#endif
7498	)
7499	{
7500	PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7501	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7502	Assert(cMsrs == 0 \|\| pMsr != NULL);
7503	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
7504	{
7505	switch (pMsr->u32Msr)
7506	{
7507	#if HC_ARCH_BITS == 32
7508	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
7509	case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
7510	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
7511	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
7512	#endif
7513	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
7514	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
7515	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7516
7517	default:
7518	{
7519	pVCpu->hm.s.u32HMError = pMsr->u32Msr;
7520	ASMSetFlags(fEFlags);
7521	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
7522	cMsrs));
7523	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7524	}
7525	}
7526	}
7527	}
7528
7529	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7530	{
7531	uint64_t u64Shadow;
7532	if (fWhat & CPUMCTX_EXTRN_CR0)
7533	{
7534	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7535	* remove when we drop 32-bit host w/ 64-bit host support, see
7536	* @bugref{9180#c39}. */
7537	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7538	#if HC_ARCH_BITS == 32
7539	uint32_t u32Shadow;
7540	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
7541	u64Shadow = u32Shadow;
7542	#else
7543	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
7544	#endif
7545	VMXLOCAL_BREAK_RC(rc);
7546	u64Val = u32Val;
7547	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7548	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7549	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7550	CPUMSetGuestCR0(pVCpu, u64Val);
7551	VMMRZCallRing3Enable(pVCpu);
7552	}
7553
7554	if (fWhat & CPUMCTX_EXTRN_CR4)
7555	{
7556	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7557	* remove when we drop 32-bit host w/ 64-bit host support, see
7558	* @bugref{9180#c39}. */
7559	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7560	#if HC_ARCH_BITS == 32
7561	uint32_t u32Shadow;
7562	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
7563	u64Shadow = u32Shadow;
7564	#else
7565	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
7566	#endif
7567	VMXLOCAL_BREAK_RC(rc);
7568	u64Val = u32Val;
7569	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
7570	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7571	pCtx->cr4 = u64Val;
7572	}
7573
7574	if (fWhat & CPUMCTX_EXTRN_CR3)
7575	{
7576	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7577	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7578	\|\| ( pVM->hm.s.fNestedPaging
7579	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7580	{
7581	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
7582	if (pCtx->cr3 != u64Val)
7583	{
7584	pCtx->cr3 = u64Val;
7585	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7586	}
7587
7588	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7589	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7590	if (CPUMIsGuestInPAEModeEx(pCtx))
7591	{
7592	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
7593	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
7594	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
7595	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
7596	VMXLOCAL_BREAK_RC(rc);
7597	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7598	}
7599	}
7600	}
7601	}
7602	} while (0);
7603
7604	if (RT_SUCCESS(rc))
7605	{
7606	/* Update fExtrn. */
7607	pCtx->fExtrn &= ~fWhat;
7608
7609	/* If everything has been imported, clear the HM keeper bit. */
7610	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7611	{
7612	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7613	Assert(!pCtx->fExtrn);
7614	}
7615	}
7616	}
7617	else
7618	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7619
7620	ASMSetFlags(fEFlags);
7621
7622	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7623
7624	if (RT_SUCCESS(rc))
7625	{ /* likely */ }
7626	else
7627	return rc;
7628
7629	/*
7630	* Honor any pending CR3 updates.
7631	*
7632	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
7633	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7634	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7635	*
7636	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7637	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7638	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7639	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7640	*
7641	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7642	*/
7643	if (VMMRZCallRing3IsEnabled(pVCpu))
7644	{
7645	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7646	{
7647	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7648	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7649	}
7650
7651	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7652	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7653
7654	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7655	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7656	}
7657
7658	return VINF_SUCCESS;
7659	#undef VMXLOCAL_BREAK_RC
7660	}
7661
7662
7663	/**
7664	* Saves the guest state from the VMCS into the guest-CPU context.
7665	*
7666	* @returns VBox status code.
7667	* @param pVCpu The cross context virtual CPU structure.
7668	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7669	*/
7670	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
7671	{
7672	PCVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7673	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7674	}
7675
7676
7677	/**
7678	* Check per-VM and per-VCPU force flag actions that require us to go back to
7679	* ring-3 for one reason or another.
7680	*
7681	* @returns Strict VBox status code (i.e. informational status codes too)
7682	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7683	* ring-3.
7684	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7685	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7686	* interrupts)
7687	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7688	* all EMTs to be in ring-3.
7689	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7690	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7691	* to the EM loop.
7692	*
7693	* @param pVCpu The cross context virtual CPU structure.
7694	* @param fStepping Whether we are single-stepping the guest using the
7695	* hypervisor debugger.
7696	*/
7697	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
7698	{
7699	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7700
7701	/*
7702	* Update pending interrupts into the APIC's IRR.
7703	*/
7704	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7705	APICUpdatePendingInterrupts(pVCpu);
7706
7707	/*
7708	* Anything pending? Should be more likely than not if we're doing a good job.
7709	*/
7710	PVM pVM = pVCpu->CTX_SUFF(pVM);
7711	if ( !fStepping
7712	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7713	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7714	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7715	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7716	return VINF_SUCCESS;
7717
7718	/* Pending PGM C3 sync. */
7719	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7720	{
7721	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7722	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7723	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7724	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7725	if (rcStrict2 != VINF_SUCCESS)
7726	{
7727	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7728	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7729	return rcStrict2;
7730	}
7731	}
7732
7733	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7734	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7735	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7736	{
7737	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7738	int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7739	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7740	return rc2;
7741	}
7742
7743	/* Pending VM request packets, such as hardware interrupts. */
7744	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7745	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7746	{
7747	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7748	return VINF_EM_PENDING_REQUEST;
7749	}
7750
7751	/* Pending PGM pool flushes. */
7752	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7753	{
7754	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7755	return VINF_PGM_POOL_FLUSH_PENDING;
7756	}
7757
7758	/* Pending DMA requests. */
7759	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7760	{
7761	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7762	return VINF_EM_RAW_TO_R3;
7763	}
7764
7765	return VINF_SUCCESS;
7766	}
7767
7768
7769	/**
7770	* Converts any TRPM trap into a pending HM event. This is typically used when
7771	* entering from ring-3 (not longjmp returns).
7772	*
7773	* @param pVCpu The cross context virtual CPU structure.
7774	*/
7775	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7776	{
7777	Assert(TRPMHasTrap(pVCpu));
7778	Assert(!pVCpu->hm.s.Event.fPending);
7779
7780	uint8_t uVector;
7781	TRPMEVENT enmTrpmEvent;
7782	RTGCUINT uErrCode;
7783	RTGCUINTPTR GCPtrFaultAddress;
7784	uint8_t cbInstr;
7785
7786	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7787	AssertRC(rc);
7788
7789	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7790	uint32_t u32IntInfo = uVector \| VMX_EXIT_INT_INFO_VALID;
7791	if (enmTrpmEvent == TRPM_TRAP)
7792	{
7793	/** @todo r=ramshankar: TRPM currently offers no way to determine a \#DB that was
7794	* generated using INT1 (ICEBP). */
7795	switch (uVector)
7796	{
7797	case X86_XCPT_NMI:
7798	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7799	break;
7800
7801	case X86_XCPT_BP:
7802	case X86_XCPT_OF:
7803	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7804	break;
7805
7806	case X86_XCPT_PF:
7807	case X86_XCPT_DF:
7808	case X86_XCPT_TS:
7809	case X86_XCPT_NP:
7810	case X86_XCPT_SS:
7811	case X86_XCPT_GP:
7812	case X86_XCPT_AC:
7813	u32IntInfo \|= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7814	RT_FALL_THRU();
7815	default:
7816	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7817	break;
7818	}
7819	}
7820	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7821	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7822	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7823	{
7824	switch (uVector)
7825	{
7826	case X86_XCPT_BP:
7827	case X86_XCPT_OF:
7828	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7829	break;
7830
7831	default:
7832	Assert(uVector == X86_XCPT_DB);
7833	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7834	break;
7835	}
7836	}
7837	else
7838	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7839
7840	rc = TRPMResetTrap(pVCpu);
7841	AssertRC(rc);
7842	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7843	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7844
7845	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7846	}
7847
7848
7849	/**
7850	* Converts the pending HM event into a TRPM trap.
7851	*
7852	* @param pVCpu The cross context virtual CPU structure.
7853	*/
7854	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7855	{
7856	Assert(pVCpu->hm.s.Event.fPending);
7857
7858	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7859	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7860	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
7861	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7862
7863	/* If a trap was already pending, we did something wrong! */
7864	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7865
7866	/** @todo Use HMVmxEventToTrpmEventType() later. */
7867	TRPMEVENT enmTrapType;
7868	switch (uVectorType)
7869	{
7870	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7871	enmTrapType = TRPM_HARDWARE_INT;
7872	break;
7873
7874	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7875	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7876	enmTrapType = TRPM_TRAP;
7877	break;
7878
7879	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT: /* #DB (INT1/ICEBP). */
7880	Assert(uVector == X86_XCPT_DB);
7881	enmTrapType = TRPM_SOFTWARE_INT;
7882	break;
7883
7884	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP (INT3) and #OF (INTO) */
7885	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7886	enmTrapType = TRPM_SOFTWARE_INT;
7887	break;
7888
7889	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7890	enmTrapType = TRPM_SOFTWARE_INT;
7891	break;
7892
7893	default:
7894	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7895	enmTrapType = TRPM_32BIT_HACK;
7896	break;
7897	}
7898
7899	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7900
7901	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7902	AssertRC(rc);
7903
7904	if (fErrorCodeValid)
7905	TRPMSetErrorCode(pVCpu, uErrorCode);
7906
7907	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7908	&& uVector == X86_XCPT_PF)
7909	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7910	else if (enmTrapType == TRPM_SOFTWARE_INT)
7911	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7912
7913	/* We're now done converting the pending event. */
7914	pVCpu->hm.s.Event.fPending = false;
7915	}
7916
7917
7918	/**
7919	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7920	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7921	*
7922	* @param pVCpu The cross context virtual CPU structure.
7923	* @param pVmcsInfo The VMCS info. object.
7924	*/
7925	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
7926	{
7927	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7928	{
7929	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7930	{
7931	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7932	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7933	AssertRC(rc);
7934	}
7935	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
7936	}
7937
7938
7939	/**
7940	* Clears the interrupt-window exiting control in the VMCS.
7941	*
7942	* @param pVmcsInfo The VMCS info. object.
7943	*/
7944	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7945	{
7946	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7947	{
7948	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7949	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7950	}
7951	return VINF_SUCCESS;
7952	}
7953
7954
7955	/**
7956	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7957	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7958	*
7959	* @param pVCpu The cross context virtual CPU structure.
7960	* @param pVmcsInfo The VMCS info. object.
7961	*/
7962	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
7963	{
7964	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7965	{
7966	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7967	{
7968	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7969	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7970	AssertRC(rc);
7971	Log4Func(("Setup NMI-window exiting\n"));
7972	}
7973	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7974	}
7975
7976
7977	/**
7978	* Clears the NMI-window exiting control in the VMCS.
7979	*
7980	* @param pVmcsInfo The VMCS info. object.
7981	*/
7982	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7983	{
7984	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7985	{
7986	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7987	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7988	}
7989	return VINF_SUCCESS;
7990	}
7991
7992
7993	/**
7994	* Does the necessary state syncing before returning to ring-3 for any reason
7995	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7996	*
7997	* @returns VBox status code.
7998	* @param pVCpu The cross context virtual CPU structure.
7999	* @param fImportState Whether to import the guest state from the VMCS back
8000	* to the guest-CPU context.
8001	*
8002	* @remarks No-long-jmp zone!!!
8003	*/
8004	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
8005	{
8006	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8007	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8008
8009	RTCPUID idCpu = RTMpCpuId();
8010	Log4Func(("HostCpuId=%u\n", idCpu));
8011
8012	/*
8013	* !!! IMPORTANT !!!
8014	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
8015	*/
8016
8017	/* Save the guest state if necessary. */
8018	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8019	if (fImportState)
8020	{
8021	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8022	AssertRCReturn(rc, rc);
8023	}
8024
8025	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8026	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8027	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8028
8029	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8030	#ifdef VBOX_STRICT
8031	if (CPUMIsHyperDebugStateActive(pVCpu))
8032	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8033	#endif
8034	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8035	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8036	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8037
8038	#if HC_ARCH_BITS == 64
8039	/* Restore host-state bits that VT-x only restores partially. */
8040	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8041	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8042	{
8043	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8044	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8045	}
8046	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8047	#endif
8048
8049	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8050	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8051	{
8052	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8053	if (!fImportState)
8054	{
8055	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8056	AssertRCReturn(rc, rc);
8057	}
8058	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8059	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8060	}
8061	else
8062	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8063
8064	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8065	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8066
8067	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8068	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8069	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8070	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8071	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8072	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8073	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8074	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8075	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8076
8077	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8078
8079	/** @todo This partially defeats the purpose of having preemption hooks.
8080	* The problem is, deregistering the hooks should be moved to a place that
8081	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8082	* context.
8083	*/
8084	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8085	AssertRCReturn(rc, rc);
8086
8087	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8088	NOREF(idCpu);
8089	return VINF_SUCCESS;
8090	}
8091
8092
8093	/**
8094	* Leaves the VT-x session.
8095	*
8096	* @returns VBox status code.
8097	* @param pVCpu The cross context virtual CPU structure.
8098	*
8099	* @remarks No-long-jmp zone!!!
8100	*/
8101	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8102	{
8103	HM_DISABLE_PREEMPT(pVCpu);
8104	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8105	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8106	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8107
8108	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8109	and done this from the VMXR0ThreadCtxCallback(). */
8110	if (!pVCpu->hm.s.fLeaveDone)
8111	{
8112	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8113	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8114	pVCpu->hm.s.fLeaveDone = true;
8115	}
8116	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8117
8118	/*
8119	* !!! IMPORTANT !!!
8120	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8121	*/
8122
8123	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8124	/** @todo Deregistering here means we need to VMCLEAR always
8125	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8126	* for calling VMMR0ThreadCtxHookDisable here! */
8127	VMMR0ThreadCtxHookDisable(pVCpu);
8128
8129	/* Leave HM context. This takes care of local init (term). */
8130	int rc = HMR0LeaveCpu(pVCpu);
8131
8132	HM_RESTORE_PREEMPT();
8133	return rc;
8134	}
8135
8136
8137	/**
8138	* Does the necessary state syncing before doing a longjmp to ring-3.
8139	*
8140	* @returns VBox status code.
8141	* @param pVCpu The cross context virtual CPU structure.
8142	*
8143	* @remarks No-long-jmp zone!!!
8144	*/
8145	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8146	{
8147	return hmR0VmxLeaveSession(pVCpu);
8148	}
8149
8150
8151	/**
8152	* Take necessary actions before going back to ring-3.
8153	*
8154	* An action requires us to go back to ring-3. This function does the necessary
8155	* steps before we can safely return to ring-3. This is not the same as longjmps
8156	* to ring-3, this is voluntary and prepares the guest so it may continue
8157	* executing outside HM (recompiler/IEM).
8158	*
8159	* @returns VBox status code.
8160	* @param pVCpu The cross context virtual CPU structure.
8161	* @param rcExit The reason for exiting to ring-3. Can be
8162	* VINF_VMM_UNKNOWN_RING3_CALL.
8163	*/
8164	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8165	{
8166	Assert(pVCpu);
8167	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8168
8169	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8170	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8171	{
8172	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8173	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8174	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8175	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8176	}
8177
8178	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8179	VMMRZCallRing3Disable(pVCpu);
8180	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8181
8182	/*
8183	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8184	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8185	*
8186	* This is because execution may continue from ring-3 and we would need to inject
8187	* the event from there (hence place it back in TRPM).
8188	*/
8189	if (pVCpu->hm.s.Event.fPending)
8190	{
8191	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8192	Assert(!pVCpu->hm.s.Event.fPending);
8193
8194	/* Clear the events from the VMCS. */
8195	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8196	AssertRCReturn(rc, rc);
8197	}
8198	#ifdef VBOX_STRICT
8199	else
8200	{
8201	/*
8202	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8203	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8204	* occasionally, see @bugref{9180#c42}.
8205	*/
8206	uint32_t uEntryIntInfo;
8207	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8208	AssertRC(rc);
8209	Assert(!VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8210	}
8211	#endif
8212
8213	/*
8214	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8215	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8216	* (e.g. TPR below threshold).
8217	*/
8218	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8219	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8220	AssertRCReturn(rc, rc);
8221
8222	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8223	and if we're injecting an event we should have a TRPM trap pending. */
8224	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8225	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8226	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8227	#endif
8228
8229	/* Save guest state and restore host state bits. */
8230	rc = hmR0VmxLeaveSession(pVCpu);
8231	AssertRCReturn(rc, rc);
8232	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8233
8234	/* Thread-context hooks are unregistered at this point!!! */
8235
8236	/* Sync recompiler state. */
8237	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8238	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8239	\| CPUM_CHANGED_LDTR
8240	\| CPUM_CHANGED_GDTR
8241	\| CPUM_CHANGED_IDTR
8242	\| CPUM_CHANGED_TR
8243	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8244	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8245	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8246	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8247
8248	Assert(!pVCpu->hm.s.fClearTrapFlag);
8249
8250	/* Update the exit-to-ring 3 reason. */
8251	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8252
8253	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8254	if ( rcExit != VINF_EM_RAW_INTERRUPT
8255	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8256	{
8257	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8258	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8259	}
8260
8261	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8262
8263	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
8264	VMMRZCallRing3RemoveNotification(pVCpu);
8265	VMMRZCallRing3Enable(pVCpu);
8266
8267	return rc;
8268	}
8269
8270
8271	/**
8272	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8273	* longjump to ring-3 and possibly get preempted.
8274	*
8275	* @returns VBox status code.
8276	* @param pVCpu The cross context virtual CPU structure.
8277	* @param enmOperation The operation causing the ring-3 longjump.
8278	* @param pvUser User argument, currently unused, NULL.
8279	*/
8280	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
8281	{
8282	RT_NOREF(pvUser);
8283	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8284	{
8285	/*
8286	* !!! IMPORTANT !!!
8287	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8288	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8289	*/
8290	VMMRZCallRing3RemoveNotification(pVCpu);
8291	VMMRZCallRing3Disable(pVCpu);
8292	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
8293	RTThreadPreemptDisable(&PreemptState);
8294
8295	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8296	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8297	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8298	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8299
8300	#if HC_ARCH_BITS == 64
8301	/* Restore host-state bits that VT-x only restores partially. */
8302	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8303	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8304	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8305	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8306	#endif
8307
8308	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8309	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8310	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8311
8312	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8313	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8314	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8315
8316	/* Clear the current VMCS data back to memory. */
8317	hmR0VmxClearVmcs(pVmcsInfo);
8318
8319	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8320	VMMR0ThreadCtxHookDisable(pVCpu);
8321	HMR0LeaveCpu(pVCpu);
8322	RTThreadPreemptRestore(&PreemptState);
8323	return VINF_SUCCESS;
8324	}
8325
8326	Assert(pVCpu);
8327	Assert(pvUser);
8328	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8329	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8330
8331	VMMRZCallRing3Disable(pVCpu);
8332	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8333
8334	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8335
8336	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8337	AssertRCReturn(rc, rc);
8338
8339	VMMRZCallRing3Enable(pVCpu);
8340	return VINF_SUCCESS;
8341	}
8342
8343
8344	/**
8345	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8346	* stack.
8347	*
8348	* @returns Strict VBox status code (i.e. informational status codes too).
8349	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8350	* @param pVCpu The cross context virtual CPU structure.
8351	* @param uValue The value to push to the guest stack.
8352	*/
8353	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
8354	{
8355	/*
8356	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8357	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8358	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8359	*/
8360	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8361	if (pCtx->sp == 1)
8362	return VINF_EM_RESET;
8363	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8364	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8365	AssertRC(rc);
8366	return rc;
8367	}
8368
8369
8370	/**
8371	* Injects an event into the guest upon VM-entry by updating the relevant fields
8372	* in the VM-entry area in the VMCS.
8373	*
8374	* @returns Strict VBox status code (i.e. informational status codes too).
8375	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8376	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8377	*
8378	* @param pVCpu The cross context virtual CPU structure.
8379	* @param pVmxTransient The VMX-transient structure.
8380	* @param pEvent The event being injected.
8381	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state.
8382	* This will be updated if necessary. This cannot not
8383	* be NULL.
8384	* @param fStepping Whether we're single-stepping guest execution and
8385	* should return VINF_EM_DBG_STEPPED if the event is
8386	* injected directly (registers modified by us, not by
8387	* hardware on VM-entry).
8388	*/
8389	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8390	uint32_t *pfIntrState)
8391	{
8392	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8393	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8394	Assert(pfIntrState);
8395
8396	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8397	uint32_t u32IntInfo = pEvent->u64IntInfo;
8398	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8399	uint32_t const cbInstr = pEvent->cbInstr;
8400	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8401	uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8402	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8403
8404	#ifdef VBOX_STRICT
8405	/*
8406	* Validate the error-code-valid bit for hardware exceptions.
8407	* No error codes for exceptions in real-mode.
8408	*
8409	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8410	*/
8411	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8412	&& !CPUMIsGuestInRealModeEx(pCtx))
8413	{
8414	switch (uVector)
8415	{
8416	case X86_XCPT_PF:
8417	case X86_XCPT_DF:
8418	case X86_XCPT_TS:
8419	case X86_XCPT_NP:
8420	case X86_XCPT_SS:
8421	case X86_XCPT_GP:
8422	case X86_XCPT_AC:
8423	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8424	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8425	RT_FALL_THRU();
8426	default:
8427	break;
8428	}
8429	}
8430
8431	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8432	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8433	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8434	#endif
8435
8436	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8437
8438	/*
8439	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8440	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8441	* interrupt handler in the (real-mode) guest.
8442	*
8443	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8444	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8445	*/
8446	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8447	{
8448	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8449	{
8450	/*
8451	* For CPUs with unrestricted guest execution enabled and with the guest
8452	* in real-mode, we must not set the deliver-error-code bit.
8453	*
8454	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8455	*/
8456	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8457	}
8458	else
8459	{
8460	PVM pVM = pVCpu->CTX_SUFF(pVM);
8461	Assert(PDMVmmDevHeapIsEnabled(pVM));
8462	Assert(pVM->hm.s.vmx.pRealModeTSS);
8463	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8464
8465	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8466	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8467	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8468	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8469	AssertRCReturn(rc2, rc2);
8470
8471	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8472	size_t const cbIdtEntry = sizeof(X86IDTR16);
8473	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8474	{
8475	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8476	if (uVector == X86_XCPT_DF)
8477	return VINF_EM_RESET;
8478
8479	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8480	No error codes for exceptions in real-mode. */
8481	if (uVector == X86_XCPT_GP)
8482	{
8483	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8484	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8485	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8486	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8487	HMEVENT EventXcptDf;
8488	RT_ZERO(EventXcptDf);
8489	EventXcptDf.u64IntInfo = uXcptDfInfo;
8490	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8491	}
8492
8493	/*
8494	* If we're injecting an event with no valid IDT entry, inject a #GP.
8495	* No error codes for exceptions in real-mode.
8496	*
8497	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8498	*/
8499	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8500	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8501	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8502	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8503	HMEVENT EventXcptGp;
8504	RT_ZERO(EventXcptGp);
8505	EventXcptGp.u64IntInfo = uXcptGpInfo;
8506	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8507	}
8508
8509	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8510	uint16_t uGuestIp = pCtx->ip;
8511	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8512	{
8513	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8514	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8515	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8516	}
8517	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8518	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8519
8520	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8521	X86IDTR16 IdtEntry;
8522	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8523	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8524	AssertRCReturn(rc2, rc2);
8525
8526	/* Construct the stack frame for the interrupt/exception handler. */
8527	VBOXSTRICTRC rcStrict;
8528	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8529	if (rcStrict == VINF_SUCCESS)
8530	{
8531	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8532	if (rcStrict == VINF_SUCCESS)
8533	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8534	}
8535
8536	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8537	if (rcStrict == VINF_SUCCESS)
8538	{
8539	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8540	pCtx->rip = IdtEntry.offSel;
8541	pCtx->cs.Sel = IdtEntry.uSel;
8542	pCtx->cs.ValidSel = IdtEntry.uSel;
8543	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8544	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8545	&& uVector == X86_XCPT_PF)
8546	pCtx->cr2 = GCPtrFault;
8547
8548	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8549	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8550	\| HM_CHANGED_GUEST_RSP);
8551
8552	/*
8553	* If we delivered a hardware exception (other than an NMI) and if there was
8554	* block-by-STI in effect, we should clear it.
8555	*/
8556	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8557	{
8558	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8559	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8560	Log4Func(("Clearing inhibition due to STI\n"));
8561	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8562	}
8563
8564	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8565	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8566
8567	/*
8568	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8569	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8570	*/
8571	pVCpu->hm.s.Event.fPending = false;
8572
8573	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8574	if (fStepping)
8575	rcStrict = VINF_EM_DBG_STEPPED;
8576	}
8577	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8578	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8579	return rcStrict;
8580	}
8581	}
8582
8583	/*
8584	* Validate.
8585	*/
8586	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8587	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8588
8589	/*
8590	* Inject the event into the VMCS.
8591	*/
8592	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8593	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8594	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8595	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8596	AssertRCReturn(rc, rc);
8597
8598	/*
8599	* Update guest CR2 if this is a page-fault.
8600	*/
8601	if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8602	&& uVector == X86_XCPT_PF)
8603	pCtx->cr2 = GCPtrFault;
8604
8605	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8606	return VINF_SUCCESS;
8607	}
8608
8609
8610	/**
8611	* Evaluates the event to be delivered to the guest and sets it as the pending
8612	* event.
8613	*
8614	* @returns Strict VBox status code (i.e. informational status codes too).
8615	* @param pVCpu The cross context virtual CPU structure.
8616	* @param pVmxTransient The VMX-transient structure.
8617	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8618	*/
8619	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8620	{
8621	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8622	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8623
8624	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
8625	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmcsInfo);
8626	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8627	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8628	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8629
8630	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8631	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8632	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8633	Assert(!TRPMHasTrap(pVCpu));
8634	Assert(pfIntrState);
8635
8636	*pfIntrState = fIntrState;
8637
8638	/*
8639	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
8640	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
8641	*/
8642	/** @todo SMI. SMIs take priority over NMIs. */
8643	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
8644	{
8645	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8646	if ( !pVCpu->hm.s.Event.fPending
8647	&& !fBlockNmi
8648	&& !fBlockSti
8649	&& !fBlockMovSS)
8650	{
8651	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8652	if ( pVmxTransient->fIsNestedGuest
8653	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8654	return IEMExecVmxVmexitNmi(pVCpu);
8655	#endif
8656	hmR0VmxSetPendingXcptNmi(pVCpu);
8657	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8658	Log4Func(("Pending NMI\n"));
8659	}
8660	else
8661	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8662	}
8663	/*
8664	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
8665	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
8666	*/
8667	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8668	&& !pVCpu->hm.s.fSingleInstruction)
8669	{
8670	Assert(!DBGFIsStepping(pVCpu));
8671	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8672	AssertRCReturn(rc, rc);
8673	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8674	if ( !pVCpu->hm.s.Event.fPending
8675	&& !fBlockInt
8676	&& !fBlockSti
8677	&& !fBlockMovSS)
8678	{
8679	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8680	if ( pVmxTransient->fIsNestedGuest
8681	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
8682	{
8683	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector /, true / fIntPending */);
8684	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8685	return rcStrict;
8686	}
8687	#endif
8688	uint8_t u8Interrupt;
8689	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8690	if (RT_SUCCESS(rc))
8691	{
8692	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8693	if ( pVmxTransient->fIsNestedGuest
8694	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8695	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8696	{
8697	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8698	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8699	return rcStrict;
8700	}
8701	#endif
8702	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8703	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
8704	}
8705	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8706	{
8707	if ( !pVmxTransient->fIsNestedGuest
8708	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8709	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
8710	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8711
8712	/*
8713	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8714	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8715	* need to re-set this force-flag here.
8716	*/
8717	}
8718	else
8719	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8720	}
8721	else
8722	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8723	}
8724
8725	return VINF_SUCCESS;
8726	}
8727
8728
8729	/**
8730	* Injects any pending events into the guest if the guest is in a state to
8731	* receive them.
8732	*
8733	* @returns Strict VBox status code (i.e. informational status codes too).
8734	* @param pVCpu The cross context virtual CPU structure.
8735	* @param pVmxTransient The VMX-transient structure.
8736	* @param fIntrState The VT-x guest-interruptibility state.
8737	* @param fStepping Whether we are single-stepping the guest using the
8738	* hypervisor debugger and should return
8739	* VINF_EM_DBG_STEPPED if the event was dispatched
8740	* directly.
8741	*/
8742	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
8743	{
8744	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8745	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8746
8747	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8748	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8749
8750	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8751	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8752	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8753	Assert(!TRPMHasTrap(pVCpu));
8754
8755	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8756	if (pVCpu->hm.s.Event.fPending)
8757	{
8758	/*
8759	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8760	* pending even while injecting an event and in this case, we want a VM-exit as soon as
8761	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
8762	*
8763	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8764	*/
8765	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8766	#ifdef VBOX_STRICT
8767	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8768	{
8769	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
8770	Assert(!fBlockInt);
8771	Assert(!fBlockSti);
8772	Assert(!fBlockMovSS);
8773	}
8774	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
8775	{
8776	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8777	Assert(!fBlockSti);
8778	Assert(!fBlockMovSS);
8779	Assert(!fBlockNmi);
8780	}
8781	#endif
8782	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8783	uIntType));
8784
8785	/*
8786	* Inject the event and get any changes to the guest-interruptibility state.
8787	*
8788	* The guest-interruptibility state may need to be updated if we inject the event
8789	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
8790	*/
8791	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
8792	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8793
8794	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8795	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8796	else
8797	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8798	}
8799
8800	/*
8801	* Update the guest-interruptibility state.
8802	*
8803	* This is required for the real-on-v86 software interrupt injection case above, as well as
8804	* updates to the guest state from ring-3 or IEM/REM.
8805	*/
8806	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
8807	AssertRCReturn(rc, rc);
8808
8809	/*
8810	* There's no need to clear the VM-entry interruption-information field here if we're not
8811	* injecting anything. VT-x clears the valid bit on every VM-exit.
8812	*
8813	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8814	*/
8815
8816	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8817	NOREF(fBlockMovSS); NOREF(fBlockSti);
8818	return rcStrict;
8819	}
8820
8821
8822	/**
8823	* Enters the VT-x session.
8824	*
8825	* @returns VBox status code.
8826	* @param pVCpu The cross context virtual CPU structure.
8827	*/
8828	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
8829	{
8830	AssertPtr(pVCpu);
8831	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8832	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8833
8834	LogFlowFunc(("pVCpu=%p\n", pVCpu));
8835	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8836	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8837
8838	#ifdef VBOX_STRICT
8839	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8840	RTCCUINTREG uHostCR4 = ASMGetCR4();
8841	if (!(uHostCR4 & X86_CR4_VMXE))
8842	{
8843	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8844	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8845	}
8846	#endif
8847
8848	/*
8849	* Load the appropriate VMCS as the current and active one.
8850	*/
8851	PVMXVMCSINFO pVmcsInfo;
8852	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
8853	if (!fInNestedGuestMode)
8854	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
8855	else
8856	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
8857	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
8858	if (RT_SUCCESS(rc))
8859	{
8860	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
8861	pVCpu->hm.s.fLeaveDone = false;
8862	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8863
8864	/*
8865	* Do the EMT scheduled L1D flush here if needed.
8866	*/
8867	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8868	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8869	}
8870	return rc;
8871	}
8872
8873
8874	/**
8875	* The thread-context callback (only on platforms which support it).
8876	*
8877	* @param enmEvent The thread-context event.
8878	* @param pVCpu The cross context virtual CPU structure.
8879	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8880	* @thread EMT(pVCpu)
8881	*/
8882	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8883	{
8884	NOREF(fGlobalInit);
8885
8886	switch (enmEvent)
8887	{
8888	case RTTHREADCTXEVENT_OUT:
8889	{
8890	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8891	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8892	VMCPU_ASSERT_EMT(pVCpu);
8893
8894	/* No longjmps (logger flushes, locks) in this fragile context. */
8895	VMMRZCallRing3Disable(pVCpu);
8896	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8897
8898	/* Restore host-state (FPU, debug etc.) */
8899	if (!pVCpu->hm.s.fLeaveDone)
8900	{
8901	/*
8902	* Do -not- import the guest-state here as we might already be in the middle of importing
8903	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8904	*/
8905	hmR0VmxLeave(pVCpu, false /* fImportState */);
8906	pVCpu->hm.s.fLeaveDone = true;
8907	}
8908
8909	/* Leave HM context, takes care of local init (term). */
8910	int rc = HMR0LeaveCpu(pVCpu);
8911	AssertRC(rc);
8912
8913	/* Restore longjmp state. */
8914	VMMRZCallRing3Enable(pVCpu);
8915	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8916	break;
8917	}
8918
8919	case RTTHREADCTXEVENT_IN:
8920	{
8921	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8922	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8923	VMCPU_ASSERT_EMT(pVCpu);
8924
8925	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8926	VMMRZCallRing3Disable(pVCpu);
8927	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8928
8929	/* Initialize the bare minimum state required for HM. This takes care of
8930	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8931	int rc = hmR0EnterCpu(pVCpu);
8932	AssertRC(rc);
8933	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8934	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8935
8936	/* Load the active VMCS as the current one. */
8937	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8938	rc = hmR0VmxLoadVmcs(pVmcsInfo);
8939	AssertRC(rc);
8940	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8941	pVCpu->hm.s.fLeaveDone = false;
8942
8943	/* Do the EMT scheduled L1D flush if needed. */
8944	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8945	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8946
8947	/* Restore longjmp state. */
8948	VMMRZCallRing3Enable(pVCpu);
8949	break;
8950	}
8951
8952	default:
8953	break;
8954	}
8955	}
8956
8957
8958	/**
8959	* Exports the host state into the VMCS host-state area.
8960	* Sets up the VM-exit MSR-load area.
8961	*
8962	* The CPU state will be loaded from these fields on every successful VM-exit.
8963	*
8964	* @returns VBox status code.
8965	* @param pVCpu The cross context virtual CPU structure.
8966	*
8967	* @remarks No-long-jump zone!!!
8968	*/
8969	static int hmR0VmxExportHostState(PVMCPU pVCpu)
8970	{
8971	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8972
8973	int rc = VINF_SUCCESS;
8974	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8975	{
8976	rc = hmR0VmxExportHostControlRegs();
8977	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8978
8979	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8980	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8981
8982	rc = hmR0VmxExportHostMsrs(pVCpu);
8983	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8984
8985	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8986	}
8987	return rc;
8988	}
8989
8990
8991	/**
8992	* Saves the host state in the VMCS host-state.
8993	*
8994	* @returns VBox status code.
8995	* @param pVCpu The cross context virtual CPU structure.
8996	*
8997	* @remarks No-long-jump zone!!!
8998	*/
8999	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
9000	{
9001	AssertPtr(pVCpu);
9002	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9003
9004	/*
9005	* Export the host state here while entering HM context.
9006	* When thread-context hooks are used, we might get preempted and have to re-save the host
9007	* state but most of the time we won't be, so do it here before we disable interrupts.
9008	*/
9009	return hmR0VmxExportHostState(pVCpu);
9010	}
9011
9012
9013	/**
9014	* Exports the guest state into the VMCS guest-state area.
9015	*
9016	* The will typically be done before VM-entry when the guest-CPU state and the
9017	* VMCS state may potentially be out of sync.
9018	*
9019	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9020	* VM-entry controls.
9021	* Sets up the appropriate VMX non-root function to execute guest code based on
9022	* the guest CPU mode.
9023	*
9024	* @returns VBox strict status code.
9025	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9026	* without unrestricted guest execution and the VMMDev is not presently
9027	* mapped (e.g. EFI32).
9028	*
9029	* @param pVCpu The cross context virtual CPU structure.
9030	* @param pVmxTransient The VMX-transient structure.
9031	*
9032	* @remarks No-long-jump zone!!!
9033	*/
9034	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9035	{
9036	AssertPtr(pVCpu);
9037	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9038	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9039
9040	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9041
9042	/*
9043	* Determine real-on-v86 mode.
9044	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9045	*/
9046	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9047	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9048	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9049	pVmcsInfo->RealMode. fRealOnV86Active = false;
9050	else
9051	{
9052	Assert(!pVmxTransient->fIsNestedGuest);
9053	pVmcsInfo->RealMode.fRealOnV86Active = true;
9054	}
9055
9056	/*
9057	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9058	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9059	*/
9060	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9061	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9062	* be a need to evaluate this everytime since I'm pretty sure we intercept
9063	* all guest paging mode changes. */
9064	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9065	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9066
9067	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9068	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9069
9070	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9071	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9072
9073	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9074	if (rcStrict == VINF_SUCCESS)
9075	{ /* likely */ }
9076	else
9077	{
9078	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9079	return rcStrict;
9080	}
9081
9082	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9083	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9084
9085	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9086	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9087
9088	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9089	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9090
9091	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9092	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9093
9094	rc = hmR0VmxExportGuestRip(pVCpu);
9095	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9096	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9097	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9098
9099	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9100	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9101	\| HM_CHANGED_GUEST_CR2
9102	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9103	\| HM_CHANGED_GUEST_X87
9104	\| HM_CHANGED_GUEST_SSE_AVX
9105	\| HM_CHANGED_GUEST_OTHER_XSAVE
9106	\| HM_CHANGED_GUEST_XCRx
9107	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9108	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9109	\| HM_CHANGED_GUEST_TSC_AUX
9110	\| HM_CHANGED_GUEST_OTHER_MSRS
9111	\| HM_CHANGED_GUEST_HWVIRT /* More accurate PLE handling someday? */
9112	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9113
9114	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9115	return rc;
9116	}
9117
9118
9119	/**
9120	* Exports the state shared between the host and guest into the VMCS.
9121	*
9122	* @param pVCpu The cross context virtual CPU structure.
9123	* @param pVmxTransient The VMX-transient structure.
9124	*
9125	* @remarks No-long-jump zone!!!
9126	*/
9127	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9128	{
9129	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9130	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9131
9132	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9133	{
9134	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9135	AssertRC(rc);
9136	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9137
9138	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9139	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9140	{
9141	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9142	AssertRC(rc);
9143	}
9144	}
9145
9146	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9147	{
9148	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9149	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9150	}
9151
9152	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9153	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9154	}
9155
9156
9157	/**
9158	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9159	*
9160	* @returns Strict VBox status code (i.e. informational status codes too).
9161	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9162	* without unrestricted guest execution and the VMMDev is not presently
9163	* mapped (e.g. EFI32).
9164	*
9165	* @param pVCpu The cross context virtual CPU structure.
9166	* @param pVmxTransient The VMX-transient structure.
9167	*
9168	* @remarks No-long-jump zone!!!
9169	*/
9170	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9171	{
9172	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9173	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9174	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9175
9176	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9177	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9178	#endif
9179
9180	/*
9181	* For many exits it's only RIP that changes and hence try to export it first
9182	* without going through a lot of change flag checks.
9183	*/
9184	VBOXSTRICTRC rcStrict;
9185	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9186	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9187	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9188	{
9189	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9190	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9191	{ /* likely */}
9192	else
9193	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9194	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9195	}
9196	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9197	{
9198	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9199	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9200	{ /* likely */}
9201	else
9202	{
9203	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9204	VBOXSTRICTRC_VAL(rcStrict)));
9205	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9206	return rcStrict;
9207	}
9208	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9209	}
9210	else
9211	rcStrict = VINF_SUCCESS;
9212
9213	#ifdef VBOX_STRICT
9214	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9215	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9216	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9217	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9218	("fCtxChanged=%#RX64\n", fCtxChanged));
9219	#endif
9220	return rcStrict;
9221	}
9222
9223
9224	/**
9225	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9226	* and update error record fields accordingly.
9227	*
9228	* @return VMX_IGS_* return codes.
9229	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9230	* wrong with the guest state.
9231	*
9232	* @param pVCpu The cross context virtual CPU structure.
9233	* @param pVmcsInfo The VMCS info. object.
9234	*
9235	* @remarks This function assumes our cache of the VMCS controls
9236	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9237	*/
9238	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
9239	{
9240	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9241	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9242	uError = (err); \
9243	break; \
9244	} else do { } while (0)
9245
9246	int rc;
9247	PVM pVM = pVCpu->CTX_SUFF(pVM);
9248	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9249	uint32_t uError = VMX_IGS_ERROR;
9250	uint32_t u32Val;
9251	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9252
9253	do
9254	{
9255	/*
9256	* CR0.
9257	*/
9258	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9259	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9260	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
9261	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9262	if (fUnrestrictedGuest)
9263	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
9264
9265	uint32_t u32GuestCr0;
9266	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
9267	AssertRCBreak(rc);
9268	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9269	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9270	if ( !fUnrestrictedGuest
9271	&& (u32GuestCr0 & X86_CR0_PG)
9272	&& !(u32GuestCr0 & X86_CR0_PE))
9273	{
9274	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9275	}
9276
9277	/*
9278	* CR4.
9279	*/
9280	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9281	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9282
9283	uint32_t u32GuestCr4;
9284	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
9285	AssertRCBreak(rc);
9286	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9287	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9288
9289	/*
9290	* IA32_DEBUGCTL MSR.
9291	*/
9292	uint64_t u64Val;
9293	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9294	AssertRCBreak(rc);
9295	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9296	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9297	{
9298	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9299	}
9300	uint64_t u64DebugCtlMsr = u64Val;
9301
9302	#ifdef VBOX_STRICT
9303	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9304	AssertRCBreak(rc);
9305	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9306	#endif
9307	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9308
9309	/*
9310	* RIP and RFLAGS.
9311	*/
9312	uint32_t u32Eflags;
9313	#if HC_ARCH_BITS == 64
9314	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9315	AssertRCBreak(rc);
9316	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9317	if ( !fLongModeGuest
9318	\|\| !pCtx->cs.Attr.n.u1Long)
9319	{
9320	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9321	}
9322	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9323	* must be identical if the "IA-32e mode guest" VM-entry
9324	* control is 1 and CS.L is 1. No check applies if the
9325	* CPU supports 64 linear-address bits. */
9326
9327	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9328	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9329	AssertRCBreak(rc);
9330	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9331	VMX_IGS_RFLAGS_RESERVED);
9332	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9333	u32Eflags = u64Val;
9334	#else
9335	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9336	AssertRCBreak(rc);
9337	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9338	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9339	#endif
9340
9341	if ( fLongModeGuest
9342	\|\| ( fUnrestrictedGuest
9343	&& !(u32GuestCr0 & X86_CR0_PE)))
9344	{
9345	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9346	}
9347
9348	uint32_t u32EntryInfo;
9349	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9350	AssertRCBreak(rc);
9351	if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9352	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9353	{
9354	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9355	}
9356
9357	/*
9358	* 64-bit checks.
9359	*/
9360	#if HC_ARCH_BITS == 64
9361	if (fLongModeGuest)
9362	{
9363	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9364	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9365	}
9366
9367	if ( !fLongModeGuest
9368	&& (u32GuestCr4 & X86_CR4_PCIDE))
9369	{
9370	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9371	}
9372
9373	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9374	* 51:32 beyond the processor's physical-address width are 0. */
9375
9376	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9377	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9378	{
9379	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9380	}
9381
9382	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9383	AssertRCBreak(rc);
9384	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9385
9386	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9387	AssertRCBreak(rc);
9388	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9389	#endif
9390
9391	/*
9392	* PERF_GLOBAL MSR.
9393	*/
9394	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9395	{
9396	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9397	AssertRCBreak(rc);
9398	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9399	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9400	}
9401
9402	/*
9403	* PAT MSR.
9404	*/
9405	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9406	{
9407	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9408	AssertRCBreak(rc);
9409	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9410	for (unsigned i = 0; i < 8; i++)
9411	{
9412	uint8_t u8Val = (u64Val & 0xff);
9413	if ( u8Val != 0 /* UC */
9414	&& u8Val != 1 /* WC */
9415	&& u8Val != 4 /* WT */
9416	&& u8Val != 5 /* WP */
9417	&& u8Val != 6 /* WB */
9418	&& u8Val != 7 /* UC- */)
9419	{
9420	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9421	}
9422	u64Val >>= 8;
9423	}
9424	}
9425
9426	/*
9427	* EFER MSR.
9428	*/
9429	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9430	{
9431	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9432	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9433	AssertRCBreak(rc);
9434	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9435	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9436	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9437	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9438	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9439	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9440	* iemVmxVmentryCheckGuestState(). */
9441	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9442	\|\| !(u32GuestCr0 & X86_CR0_PG)
9443	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9444	VMX_IGS_EFER_LMA_LME_MISMATCH);
9445	}
9446
9447	/*
9448	* Segment registers.
9449	*/
9450	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9451	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9452	if (!(u32Eflags & X86_EFL_VM))
9453	{
9454	/* CS */
9455	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9456	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9457	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9458	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9459	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9460	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9461	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9462	/* CS cannot be loaded with NULL in protected mode. */
9463	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9464	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9465	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9466	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9467	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9468	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9469	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9470	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9471	else
9472	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9473
9474	/* SS */
9475	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9476	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9477	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9478	if ( !(pCtx->cr0 & X86_CR0_PE)
9479	\|\| pCtx->cs.Attr.n.u4Type == 3)
9480	{
9481	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9482	}
9483	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9484	{
9485	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9486	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9487	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9488	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9489	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9490	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9491	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9492	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9493	}
9494
9495	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9496	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9497	{
9498	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9499	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9500	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9501	\|\| pCtx->ds.Attr.n.u4Type > 11
9502	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9503	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9504	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9505	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9506	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9507	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9508	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9509	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9510	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9511	}
9512	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9513	{
9514	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9515	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9516	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9517	\|\| pCtx->es.Attr.n.u4Type > 11
9518	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9519	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9520	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9521	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9522	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9523	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9524	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9525	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9526	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9527	}
9528	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9529	{
9530	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9531	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9532	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9533	\|\| pCtx->fs.Attr.n.u4Type > 11
9534	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9535	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9536	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9537	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9538	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9539	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9540	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9541	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9542	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9543	}
9544	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9545	{
9546	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9547	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9548	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9549	\|\| pCtx->gs.Attr.n.u4Type > 11
9550	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9551	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9552	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9553	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9554	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9555	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9556	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9557	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9558	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9559	}
9560	/* 64-bit capable CPUs. */
9561	#if HC_ARCH_BITS == 64
9562	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9563	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9564	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9565	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9566	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9567	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9568	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9569	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9570	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9571	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9572	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9573	#endif
9574	}
9575	else
9576	{
9577	/* V86 mode checks. */
9578	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9579	if (pVmcsInfo->RealMode.fRealOnV86Active)
9580	{
9581	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9582	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9583	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9584	}
9585	else
9586	{
9587	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9588	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9589	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9590	}
9591
9592	/* CS */
9593	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9594	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9595	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9596	/* SS */
9597	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9598	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9599	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9600	/* DS */
9601	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9602	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9603	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9604	/* ES */
9605	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9606	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9607	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9608	/* FS */
9609	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9610	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9611	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9612	/* GS */
9613	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9614	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9615	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9616	/* 64-bit capable CPUs. */
9617	#if HC_ARCH_BITS == 64
9618	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9619	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9620	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9621	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9622	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9623	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9624	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9625	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9626	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9627	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9628	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9629	#endif
9630	}
9631
9632	/*
9633	* TR.
9634	*/
9635	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9636	/* 64-bit capable CPUs. */
9637	#if HC_ARCH_BITS == 64
9638	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9639	#endif
9640	if (fLongModeGuest)
9641	{
9642	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9643	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9644	}
9645	else
9646	{
9647	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9648	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9649	VMX_IGS_TR_ATTR_TYPE_INVALID);
9650	}
9651	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9652	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9653	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9654	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9655	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9656	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9657	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9658	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9659
9660	/*
9661	* GDTR and IDTR.
9662	*/
9663	#if HC_ARCH_BITS == 64
9664	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9665	AssertRCBreak(rc);
9666	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9667
9668	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9669	AssertRCBreak(rc);
9670	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9671	#endif
9672
9673	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9674	AssertRCBreak(rc);
9675	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9676
9677	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9678	AssertRCBreak(rc);
9679	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9680
9681	/*
9682	* Guest Non-Register State.
9683	*/
9684	/* Activity State. */
9685	uint32_t u32ActivityState;
9686	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9687	AssertRCBreak(rc);
9688	HMVMX_CHECK_BREAK( !u32ActivityState
9689	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9690	VMX_IGS_ACTIVITY_STATE_INVALID);
9691	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9692	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9693	uint32_t u32IntrState;
9694	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9695	AssertRCBreak(rc);
9696	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9697	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9698	{
9699	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9700	}
9701
9702	/** @todo Activity state and injecting interrupts. Left as a todo since we
9703	* currently don't use activity states but ACTIVE. */
9704
9705	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9706	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9707
9708	/* Guest interruptibility-state. */
9709	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9710	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9711	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9712	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9713	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9714	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9715	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9716	if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
9717	{
9718	if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9719	{
9720	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9721	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9722	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9723	}
9724	else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9725	{
9726	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9727	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9728	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9729	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9730	}
9731	}
9732	/** @todo Assumes the processor is not in SMM. */
9733	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9734	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9735	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9736	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9737	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9738	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9739	&& VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9740	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9741	{
9742	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
9743	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9744	}
9745
9746	/* Pending debug exceptions. */
9747	#if HC_ARCH_BITS == 64
9748	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
9749	AssertRCBreak(rc);
9750	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9751	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9752	u32Val = u64Val; /* For pending debug exceptions checks below. */
9753	#else
9754	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
9755	AssertRCBreak(rc);
9756	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9757	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9758	#endif
9759
9760	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9761	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
9762	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9763	{
9764	if ( (u32Eflags & X86_EFL_TF)
9765	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9766	{
9767	/* Bit 14 is PendingDebug.BS. */
9768	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9769	}
9770	if ( !(u32Eflags & X86_EFL_TF)
9771	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9772	{
9773	/* Bit 14 is PendingDebug.BS. */
9774	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9775	}
9776	}
9777
9778	/* VMCS link pointer. */
9779	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9780	AssertRCBreak(rc);
9781	if (u64Val != UINT64_C(0xffffffffffffffff))
9782	{
9783	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9784	/** @todo Bits beyond the processor's physical-address width MBZ. */
9785	/** @todo 32-bit located in memory referenced by value of this field (as a
9786	* physical address) must contain the processor's VMCS revision ID. */
9787	/** @todo SMM checks. */
9788	}
9789
9790	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9791	* not using nested paging? */
9792	if ( pVM->hm.s.fNestedPaging
9793	&& !fLongModeGuest
9794	&& CPUMIsGuestInPAEModeEx(pCtx))
9795	{
9796	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9797	AssertRCBreak(rc);
9798	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9799
9800	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9801	AssertRCBreak(rc);
9802	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9803
9804	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9805	AssertRCBreak(rc);
9806	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9807
9808	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9809	AssertRCBreak(rc);
9810	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9811	}
9812
9813	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9814	if (uError == VMX_IGS_ERROR)
9815	uError = VMX_IGS_REASON_NOT_FOUND;
9816	} while (0);
9817
9818	pVCpu->hm.s.u32HMError = uError;
9819	return uError;
9820
9821	#undef HMVMX_ERROR_BREAK
9822	#undef HMVMX_CHECK_BREAK
9823	}
9824
9825
9826	/**
9827	* Setup the APIC-access page for virtualizing APIC access.
9828	*
9829	* This can cause a longjumps to R3 due to the acquisition of the PGM lock, hence
9830	* this not done as part of exporting guest state, see @bugref{8721}.
9831	*
9832	* @returns VBox status code.
9833	* @param pVCpu The cross context virtual CPU structure.
9834	*/
9835	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
9836	{
9837	PVM pVM = pVCpu->CTX_SUFF(pVM);
9838	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
9839
9840	Assert(PDMHasApic(pVM));
9841	Assert(u64MsrApicBase);
9842
9843	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
9844	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
9845
9846	/* Unalias any existing mapping. */
9847	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
9848	AssertRCReturn(rc, rc);
9849
9850	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
9851	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
9852	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
9853	AssertRCReturn(rc, rc);
9854
9855	/* Update the per-VCPU cache of the APIC base MSR. */
9856	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
9857	return VINF_SUCCESS;
9858	}
9859
9860
9861	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9862	/**
9863	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
9864	* nested-guest using hardware-assisted VMX.
9865	*
9866	* @param pVCpu The cross context virtual CPU structure.
9867	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
9868	* @param pVmcsInfoGst The guest VMCS info. object.
9869	*/
9870	static void hmR0VmxMergeMsrBitmapNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
9871	{
9872	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
9873	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
9874	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
9875	Assert(pu64MsrBitmapNstGst);
9876	Assert(pu64MsrBitmapGst);
9877	Assert(pu64MsrBitmap);
9878
9879	/*
9880	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
9881	* MSR that is intercepted by the guest is also intercepted while executing the
9882	* nested-guest using hardware-assisted VMX.
9883	*/
9884	uint32_t const cbFrag = sizeof(uint64_t);
9885	uint32_t const cFrags = X86_PAGE_4K_SIZE / cbFrag;
9886	for (uint32_t i = 0; i <= cFrags; i++)
9887	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
9888	}
9889
9890
9891	/**
9892	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
9893	* hardware-assisted VMX execution of the nested-guest.
9894	*
9895	* For a guest, we don't modify these controls once we set up the VMCS.
9896	*
9897	* For nested-guests since the guest hypervisor provides these controls on every
9898	* nested-guest VM-entry and could potentially change them everytime we need to
9899	* merge them before every nested-guest VM-entry.
9900	*
9901	* @returns VBox status code.
9902	* @param pVCpu The cross context virtual CPU structure.
9903	*/
9904	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
9905	{
9906	PVM pVM = pVCpu->CTX_SUFF(pVM);
9907	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
9908	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
9909	Assert(pVmcsNstGst);
9910
9911	/*
9912	* Merge the controls with the requirements of the guest VMCS.
9913	*
9914	* We do not need to validate the nested-guest VMX features specified in the
9915	* nested-guest VMCS with the features supported by the physical CPU as it's
9916	* already done by the VMLAUNCH/VMRESUME instruction emulation.
9917	*
9918	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the
9919	* guest are derived from the VMX features supported by the physical CPU.
9920	*/
9921
9922	/* Pin-based VM-execution controls. */
9923	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
9924
9925	/* Processor-based VM-execution controls. */
9926	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
9927	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
9928	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
9929	\| VMX_PROC_CTLS_USE_TPR_SHADOW
9930	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
9931
9932	/* Secondary processor-based VM-execution controls. */
9933	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
9934	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
9935	\| VMX_PROC_CTLS2_INVPCID
9936	\| VMX_PROC_CTLS2_RDTSCP
9937	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
9938	\| VMX_PROC_CTLS2_APIC_REG_VIRT
9939	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
9940	\| VMX_PROC_CTLS2_VMFUNC));
9941
9942	/*
9943	* VM-entry controls:
9944	* These controls contains state that depends on the nested-guest state (primarily
9945	* EFER MSR) and is thus not constant through VMLAUNCH/VMRESUME and the nested-guest
9946	* VM-exit. Although the nested-hypervisor cannot change it, we need to in order to
9947	* properly continue executing the nested-guest if the EFER MSR changes but does not
9948	* cause a nested-guest VM-exits.
9949	*
9950	* VM-exit controls:
9951	* These controls specify the host state on return. We cannot use the controls from
9952	* the nested-hypervisor state as is as it would contain the guest state rather than
9953	* the host state. Since the host state is subject to change (e.g. preemption, trips
9954	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
9955	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
9956	*
9957	* VM-entry MSR-load:
9958	* The guest MSRs from the VM-entry MSR-load area are already loaded into the
9959	* guest-CPU context by the VMLAUNCH/VMRESUME instruction emulation.
9960	*
9961	* VM-exit MSR-store:
9962	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU
9963	* context back into the VM-exit MSR-store area.
9964	*
9965	* VM-exit MSR-load areas:
9966	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence,
9967	* we can entirely ignore what the nested-hypervisor wants to load here.
9968	*/
9969
9970	/*
9971	* Exception bitmap.
9972	*
9973	* We could remove #UD from the guest bitmap and merge it with the nested-guest
9974	* bitmap here (and avoid doing anything while exporting nested-guest state), but to
9975	* keep the code more flexible if intercepting exceptions become more dynamic in
9976	* the future we do it as part of exporting the nested-guest state.
9977	*/
9978	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
9979
9980	/*
9981	* CR0/CR4 guest/host mask.
9982	*
9983	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest
9984	* must cause VM-exits, so we need to merge them here.
9985	*/
9986	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
9987	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
9988
9989	/*
9990	* Page-fault error-code mask and match.
9991	*
9992	* Although we require unrestricted guest execution (and thereby nested-paging) for
9993	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
9994	* normally intercept #PFs, it might intercept them for debugging purposes.
9995	*
9996	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF
9997	* filters. If the outer guest is intercepting #PFs we must intercept all #PFs.
9998	*/
9999	uint32_t u32XcptPFMask;
10000	uint32_t u32XcptPFMatch;
10001	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10002	{
10003	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10004	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10005	}
10006	else
10007	{
10008	u32XcptPFMask = 0;
10009	u32XcptPFMatch = 0;
10010	}
10011
10012	/*
10013	* Pause-Loop exiting.
10014	*/
10015	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10016	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10017
10018	/*
10019	* I/O Bitmap.
10020	*
10021	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we
10022	* always intercept all I/O port accesses.
10023	*/
10024	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10025
10026	/*
10027	* APIC-access page.
10028	*
10029	* The APIC-access page address has already been initialized while setting up the
10030	* nested-guest VMCS. In theory, even if the guest-physical address is invalid, it
10031	* should not be on any consequence to the host or to the guest for that matter, but
10032	* we only accept valid addresses verified by the VMLAUNCH/VMRESUME instruction
10033	* emulation to keep it simple.
10034	*/
10035
10036	/*
10037	* Virtual-APIC page and TPR threshold.
10038	*
10039	* We shall use the host-physical address of the virtual-APIC page in guest memory directly.
10040	* For this reason, we can access the virtual-APIC page of the nested-guest only using
10041	* PGM physical handlers as we must not assume a kernel virtual-address mapping exists and
10042	* requesting PGM for a mapping could be expensive/resource intensive (PGM mapping cache).
10043	*/
10044	RTHCPHYS HCPhysVirtApic = NIL_RTHCPHYS;
10045	uint32_t const u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10046	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10047	{
10048	int rc = PGMPhysGCPhys2HCPhys(pVM, pVmcsNstGst->u64AddrVirtApic.u, &HCPhysVirtApic);
10049
10050	/*
10051	* If the guest hypervisor has loaded crap into the virtual-APIC page field
10052	* we would fail to obtain a valid host-physical address for its guest-physical
10053	* address.
10054	*
10055	* We currently do not support this scenario. Maybe in the future if there is a
10056	* pressing need we can explore making this particular set of conditions work.
10057	* Right now we just cause a VM-entry failure.
10058	*
10059	* This has already been checked by VMLAUNCH/VMRESUME instruction emulation,
10060	* so should not really failure at the moment.
10061	*/
10062	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10063	}
10064	else
10065	{
10066	/*
10067	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10068	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10069	* be taken care of by EPT/shadow paging.
10070	*/
10071	if (pVM->hm.s.fAllow64BitGuests)
10072	{
10073	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10074	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10075	}
10076	}
10077
10078	/*
10079	* Validate basic assumptions.
10080	*/
10081	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10082	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10083	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10084	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10085
10086	/*
10087	* Commit it to the nested-guest VMCS.
10088	*/
10089	int rc = VINF_SUCCESS;
10090	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10091	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10092	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10093	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10094	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10095	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10096	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10097	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10098	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10099	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10100	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10101	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10102	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10103	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10104	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10105	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10106	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10107	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10108	{
10109	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10110	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10111	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10112	}
10113	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10114	{
10115	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10116	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10117	}
10118	AssertRCReturn(rc, rc);
10119
10120	/*
10121	* Update the nested-guest VMCS cache.
10122	*/
10123	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10124	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10125	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10126	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10127	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10128	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10129	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10130	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10131	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10132
10133	/*
10134	* MSR bitmap.
10135	*
10136	* The MSR bitmap address has already been initialized while setting up the
10137	* nested-guest VMCS, here we need to merge the MSR bitmaps.
10138	*/
10139	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10140	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10141
10142	return VINF_SUCCESS;
10143	}
10144	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10145
10146
10147	/**
10148	* Does the preparations before executing guest code in VT-x.
10149	*
10150	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10151	* recompiler/IEM. We must be cautious what we do here regarding committing
10152	* guest-state information into the VMCS assuming we assuredly execute the
10153	* guest in VT-x mode.
10154	*
10155	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10156	* the common-state (TRPM/forceflags), we must undo those changes so that the
10157	* recompiler/IEM can (and should) use them when it resumes guest execution.
10158	* Otherwise such operations must be done when we can no longer exit to ring-3.
10159	*
10160	* @returns Strict VBox status code (i.e. informational status codes too).
10161	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10162	* have been disabled.
10163	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10164	* double-fault into the guest.
10165	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10166	* dispatched directly.
10167	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10168	*
10169	* @param pVCpu The cross context virtual CPU structure.
10170	* @param pVmxTransient The VMX-transient structure.
10171	* @param fStepping Whether we are single-stepping the guest in the
10172	* hypervisor debugger. Makes us ignore some of the reasons
10173	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10174	* if event dispatching took place.
10175	*/
10176	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10177	{
10178	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10179
10180	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10181	if (pVmxTransient->fIsNestedGuest)
10182	{
10183	RT_NOREF2(pVCpu, fStepping);
10184	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10185	return VINF_EM_RESCHEDULE_REM;
10186	}
10187	#endif
10188
10189	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10190	PGMRZDynMapFlushAutoSet(pVCpu);
10191	#endif
10192
10193	/*
10194	* Check and process force flag actions, some of which might require us to go back to ring-3.
10195	*/
10196	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10197	if (rcStrict == VINF_SUCCESS)
10198	{ /* FFs don't get set all the time. */ }
10199	else
10200	return rcStrict;
10201
10202	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10203	/*
10204	* Switch to the nested-guest VMCS as we may have transitioned into executing
10205	* the nested-guest without leaving ring-0. Otherwise, if we came from ring-3
10206	* we would load the nested-guest VMCS while entering the VMX ring-0 session.
10207	*
10208	* We do this as late as possible to minimize (though not completely remove)
10209	* clearing/loading VMCS again due to premature trips to ring-3 above.
10210	*/
10211	if (pVmxTransient->fIsNestedGuest)
10212	{
10213	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10214	{
10215	/*
10216	* Ensure we have synced everything from the guest VMCS and also flag that
10217	* that we need to export the full (nested) guest-CPU context to the
10218	* nested-guest VMCS.
10219	*/
10220	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
10221	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST);
10222
10223	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
10224	int rc = hmR0VmxSwitchVmcs(&pVCpu->hm.s.vmx.VmcsInfo, &pVCpu->hm.s.vmx.VmcsInfoNstGst);
10225	if (RT_LIKELY(rc == VINF_SUCCESS))
10226	{
10227	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = true;
10228	ASMSetFlags(fEFlags);
10229	pVmxTransient->pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10230
10231	/*
10232	* We use a different VM-exit MSR-store area for the nested-guest. Hence,
10233	* flag that we need to update the host MSR values there. Even if we decide
10234	* in the future to share the VM-exit MSR-store area page with the guest,
10235	* if its content differs, we would have to update the host MSRs anyway.
10236	*/
10237	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
10238	Assert(!pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer); /** @todo NSTVMX: Paranoia remove later. */
10239	}
10240	else
10241	{
10242	ASMSetFlags(fEFlags);
10243	return rc;
10244	}
10245	}
10246
10247	/*
10248	* Merge guest VMCS controls with the nested-guest VMCS controls.
10249	*
10250	* Even if we have not executed the guest prior to this (e.g. when resuming
10251	* from a saved state), we should be okay with merging controls as we
10252	* initialize the guest VMCS controls as part of VM setup phase.
10253	*/
10254	if (!pVCpu->hm.s.vmx.fMergedNstGstCtls)
10255	{
10256	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10257	AssertRCReturn(rc, rc);
10258	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10259	}
10260	}
10261	#endif
10262
10263	/*
10264	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10265	* We look at the guest VMCS control here as we always set it when supported by
10266	* the physical CPU. Looking at the nested-guest control here would not be
10267	* possible because they are not merged yet.
10268	*/
10269	PVM pVM = pVCpu->CTX_SUFF(pVM);
10270	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10271	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10272	&& (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10273	&& PDMHasApic(pVM))
10274	{
10275	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10276	AssertRCReturn(rc, rc);
10277	}
10278
10279	/*
10280	* Evaluate events to be injected into the guest.
10281	*
10282	* Events in TRPM can be injected without inspecting the guest state.
10283	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10284	* guest to cause a VM-exit the next time they are ready to receive the event.
10285	*/
10286	if (TRPMHasTrap(pVCpu))
10287	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10288
10289	uint32_t fIntrState;
10290	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10291
10292	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10293	/*
10294	* While evaluating pending events if something failed (unlikely) or if we were
10295	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10296	*/
10297	if ( rcStrict != VINF_SUCCESS
10298	\|\| ( pVmxTransient->fIsNestedGuest
10299	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)))
10300	return rcStrict;
10301	#endif
10302
10303	/*
10304	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10305	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10306	* also result in triple-faulting the VM.
10307	*
10308	* The above does not apply when executing a nested-guest (since unrestricted guest execution
10309	* is a requirement) regardless doing it avoid duplicating code elsewhere.
10310	*/
10311	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10312	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10313	{ /* likely */ }
10314	else
10315	{
10316	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10317	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10318	return rcStrict;
10319	}
10320
10321	/*
10322	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10323	* import CR3 themselves. We will need to update them here, as even as late as the above
10324	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10325	* the below force flags to be set.
10326	*/
10327	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10328	{
10329	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10330	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10331	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10332	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10333	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10334	}
10335	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10336	{
10337	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10338	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10339	}
10340
10341	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10342	/* Paranoia. */
10343	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10344	#endif
10345
10346	/*
10347	* No longjmps to ring-3 from this point on!!!
10348	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10349	* This also disables flushing of the R0-logger instance (if any).
10350	*/
10351	VMMRZCallRing3Disable(pVCpu);
10352
10353	/*
10354	* Export the guest state bits.
10355	*
10356	* We cannot perform longjmps while loading the guest state because we do not preserve the
10357	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10358	* CPU migration.
10359	*
10360	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10361	* registers. Hence, loading of the guest state needs to be done -after- injection of events.
10362	*/
10363	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10364	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10365	{ /* likely */ }
10366	else
10367	{
10368	VMMRZCallRing3Enable(pVCpu);
10369	return rcStrict;
10370	}
10371
10372	/*
10373	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10374	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10375	* preemption disabled for a while. Since this is purely to aid the
10376	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10377	* disable interrupt on NT.
10378	*
10379	* We need to check for force-flags that could've possible been altered since we last
10380	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10381	* see @bugref{6398}).
10382	*
10383	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10384	* to ring-3 before executing guest code.
10385	*/
10386	pVmxTransient->fEFlags = ASMIntDisableFlags();
10387
10388	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10389	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10390	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10391	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10392	{
10393	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10394	{
10395	pVCpu->hm.s.Event.fPending = false;
10396
10397	/*
10398	* We've injected any pending events. This is really the point of no return (to ring-3).
10399	*
10400	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10401	* returns from this function, so don't enable them here.
10402	*/
10403	return VINF_SUCCESS;
10404	}
10405
10406	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10407	rcStrict = VINF_EM_RAW_INTERRUPT;
10408	}
10409	else
10410	{
10411	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10412	rcStrict = VINF_EM_RAW_TO_R3;
10413	}
10414
10415	ASMSetFlags(pVmxTransient->fEFlags);
10416	VMMRZCallRing3Enable(pVCpu);
10417
10418	return rcStrict;
10419	}
10420
10421
10422	/**
10423	* Final preparations before executing guest code using hardware-assisted VMX.
10424	*
10425	* We can no longer get preempted to a different host CPU and there are no returns
10426	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10427	* failures), this function is not intended to fail sans unrecoverable hardware
10428	* errors.
10429	*
10430	* @param pVCpu The cross context virtual CPU structure.
10431	* @param pVmxTransient The VMX-transient structure.
10432	*
10433	* @remarks Called with preemption disabled.
10434	* @remarks No-long-jump zone!!!
10435	*/
10436	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10437	{
10438	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10439	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10440	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10441	Assert(!pVCpu->hm.s.Event.fPending);
10442
10443	/*
10444	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10445	*/
10446	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10447	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10448
10449	PVM pVM = pVCpu->CTX_SUFF(pVM);
10450	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10451
10452	if (!CPUMIsGuestFPUStateActive(pVCpu))
10453	{
10454	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10455	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10456	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10457	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10458	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10459	}
10460
10461	/*
10462	* Re-save the host state bits as we may've been preempted (only happens when
10463	* thread-context hooks are used or when the VM start function changes).
10464	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10465	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10466	* see @bugref{8432}.
10467	*
10468	* This may also happen when switching to/from a nested-guest VMCS without leaving
10469	* ring-0.
10470	*/
10471	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10472	{
10473	int rc = hmR0VmxExportHostState(pVCpu);
10474	AssertRC(rc);
10475	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
10476	}
10477	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10478
10479	/*
10480	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10481	*/
10482	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10483	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10484	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10485
10486	/*
10487	* Store status of the shared guest/host debug state at the time of VM-entry.
10488	*/
10489	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
10490	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
10491	{
10492	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
10493	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
10494	}
10495	else
10496	#endif
10497	{
10498	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10499	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10500	}
10501
10502	/*
10503	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10504	* more than one conditional check. The post-run side of our code shall determine
10505	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10506	*/
10507	if (pVmcsInfo->pbVirtApic)
10508	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10509
10510	/*
10511	* Update the host MSRs values in the VM-exit MSR-load area.
10512	*/
10513	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10514	{
10515	if (pVmcsInfo->cExitMsrLoad > 0)
10516	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10517	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10518	}
10519
10520	/*
10521	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10522	* VMX-preemption timer based on the next virtual sync clock deadline.
10523	*/
10524	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10525	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10526	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10527	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10528	{
10529	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10530	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10531	}
10532
10533	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10534	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10535	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10536	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
10537
10538	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10539
10540	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10541	as we're about to start executing the guest . */
10542
10543	/*
10544	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10545	*
10546	* This is done this late as updating the TSC offsetting/preemption timer above
10547	* figures out if we can skip intercepting RDTSCP by calculating the number of
10548	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10549	*/
10550	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10551	{
10552	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10553	{
10554	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10555	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10556	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10557	AssertRC(rc);
10558	}
10559	else
10560	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
10561	}
10562
10563	#ifdef VBOX_STRICT
10564	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo);
10565	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10566	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo));
10567	#endif
10568
10569	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10570	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10571	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs. */
10572	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10573	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10574	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10575	#endif
10576	}
10577
10578
10579	/**
10580	* First C routine invoked after running guest code using hardware-assisted VMX.
10581	*
10582	* @param pVCpu The cross context virtual CPU structure.
10583	* @param pVmxTransient The VMX-transient structure.
10584	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10585	*
10586	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10587	*
10588	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10589	* unconditionally when it is safe to do so.
10590	*/
10591	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10592	{
10593	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10594
10595	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10596	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10597	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10598	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10599	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10600	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10601
10602	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10603	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10604	{
10605	uint64_t uGstTsc;
10606	if (!pVmxTransient->fIsNestedGuest)
10607	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10608	else
10609	{
10610	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10611	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10612	}
10613	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10614	}
10615
10616	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10617	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
10618	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10619
10620	#if HC_ARCH_BITS == 64
10621	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10622	#endif
10623	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
10624	/* The 64-on-32 switcher maintains VMCS-launch state on its own
10625	and we need to leave it alone here. */
10626	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
10627	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10628	#else
10629	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10630	#endif
10631	#ifdef VBOX_STRICT
10632	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10633	#endif
10634	Assert(!ASMIntAreEnabled());
10635	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10636	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10637
10638	/*
10639	* Save the basic VM-exit reason and check if the VM-entry failed.
10640	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10641	*/
10642	uint32_t uExitReason;
10643	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
10644	AssertRC(rc);
10645	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
10646	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
10647
10648	/*
10649	* Check if VMLAUNCH/VMRESUME succeeded.
10650	* If this failed, we cause a guru meditation and cease further execution.
10651	*/
10652	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
10653	{
10654	/*
10655	* Update the VM-exit history array here even if the VM-entry failed due to:
10656	* - Invalid guest state.
10657	* - MSR loading.
10658	* - Machine-check event.
10659	*
10660	* In any of the above cases we will still have a "valid" VM-exit reason
10661	* despite @a fVMEntryFailed being false.
10662	*
10663	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
10664	*
10665	* Note! We don't have CS or RIP at this point. Will probably address that later
10666	* by amending the history entry added here.
10667	*/
10668	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
10669	UINT64_MAX, uHostTsc);
10670
10671	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
10672	{
10673	VMMRZCallRing3Enable(pVCpu);
10674
10675	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10676	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10677
10678	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
10679	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10680	AssertRC(rc);
10681	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
10682	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
10683	AssertRC(rc);
10684	#else
10685	/*
10686	* Import the guest-interruptibility state always as we need it while evaluating
10687	* injecting events on re-entry.
10688	*
10689	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
10690	* checking for real-mode while exporting the state because all bits that cause
10691	* mode changes wrt CR0 are intercepted.
10692	*/
10693	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
10694	AssertRC(rc);
10695	#endif
10696
10697	/*
10698	* Sync the TPR shadow with our APIC state.
10699	*/
10700	if ( !pVmxTransient->fIsNestedGuest
10701	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
10702	{
10703	Assert(pVmcsInfo->pbVirtApic);
10704	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
10705	{
10706	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
10707	AssertRC(rc);
10708	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
10709	}
10710	}
10711
10712	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10713	return;
10714	}
10715	}
10716	else
10717	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
10718
10719	VMMRZCallRing3Enable(pVCpu);
10720	}
10721
10722
10723	/**
10724	* Runs the guest code using hardware-assisted VMX the normal way.
10725	*
10726	* @returns VBox status code.
10727	* @param pVCpu The cross context virtual CPU structure.
10728	* @param pcLoops Pointer to the number of executed loops.
10729	*/
10730	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
10731	{
10732	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10733	Assert(pcLoops);
10734	Assert(*pcLoops <= cMaxResumeLoops);
10735
10736	VMXTRANSIENT VmxTransient;
10737	RT_ZERO(VmxTransient);
10738	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10739
10740	/* Paranoia. */
10741	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
10742	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10743
10744	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10745	for (;;)
10746	{
10747	Assert(!HMR0SuspendPending());
10748	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10749	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10750
10751	/*
10752	* Preparatory work for running nested-guest code, this may force us to
10753	* return to ring-3.
10754	*
10755	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10756	*/
10757	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10758	if (rcStrict != VINF_SUCCESS)
10759	break;
10760
10761	/* Interrupts are disabled at this point! */
10762	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10763	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10764	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10765	/* Interrupts are re-enabled at this point! */
10766
10767	/*
10768	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10769	*/
10770	if (RT_SUCCESS(rcRun))
10771	{ /* very likely */ }
10772	else
10773	{
10774	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10775	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10776	return rcRun;
10777	}
10778
10779	/*
10780	* Profile the VM-exit.
10781	*/
10782	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10783	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10784	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10785	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10786	HMVMX_START_EXIT_DISPATCH_PROF();
10787
10788	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10789
10790	/*
10791	* Handle the VM-exit.
10792	*/
10793	#ifdef HMVMX_USE_FUNCTION_TABLE
10794	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
10795	#else
10796	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
10797	#endif
10798	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10799	if (rcStrict == VINF_SUCCESS)
10800	{
10801	if (++(*pcLoops) <= cMaxResumeLoops)
10802	continue;
10803	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10804	rcStrict = VINF_EM_RAW_INTERRUPT;
10805	}
10806	break;
10807	}
10808
10809	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10810	return rcStrict;
10811	}
10812
10813	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10814	/**
10815	* Runs the nested-guest code using hardware-assisted VMX.
10816	*
10817	* @returns VBox status code.
10818	* @param pVCpu The cross context virtual CPU structure.
10819	* @param pcLoops Pointer to the number of executed loops.
10820	*
10821	* @sa hmR0VmxRunGuestCodeNormal().
10822	*/
10823	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
10824	{
10825	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10826	Assert(pcLoops);
10827	Assert(*pcLoops <= cMaxResumeLoops);
10828
10829	VMXTRANSIENT VmxTransient;
10830	RT_ZERO(VmxTransient);
10831	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10832	VmxTransient.fIsNestedGuest = true;
10833
10834	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10835	for (;;)
10836	{
10837	Assert(!HMR0SuspendPending());
10838	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10839	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10840
10841	/*
10842	* Preparatory work for running guest code, this may force us to
10843	* return to ring-3.
10844	*
10845	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10846	*/
10847	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10848	if (rcStrict != VINF_SUCCESS)
10849	break;
10850
10851	/* Interrupts are disabled at this point! */
10852	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10853	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10854	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10855	/* Interrupts are re-enabled at this point! */
10856
10857	/*
10858	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10859	*/
10860	if (RT_SUCCESS(rcRun))
10861	{ /* very likely */ }
10862	else
10863	{
10864	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10865	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10866	return rcRun;
10867	}
10868
10869	/*
10870	* Profile the VM-exit.
10871	*/
10872	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10873	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10874	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10875	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10876	HMVMX_START_EXIT_DISPATCH_PROF();
10877
10878	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10879
10880	/*
10881	* Handle the VM-exit.
10882	*/
10883	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
10884	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10885	if ( rcStrict == VINF_SUCCESS
10886	&& CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10887	{
10888	if (++(*pcLoops) <= cMaxResumeLoops)
10889	continue;
10890	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10891	rcStrict = VINF_EM_RAW_INTERRUPT;
10892	}
10893	break;
10894	}
10895
10896	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10897	return rcStrict;
10898	}
10899	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10900
10901
10902	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
10903	* probes.
10904	*
10905	* The following few functions and associated structure contains the bloat
10906	* necessary for providing detailed debug events and dtrace probes as well as
10907	* reliable host side single stepping. This works on the principle of
10908	* "subclassing" the normal execution loop and workers. We replace the loop
10909	* method completely and override selected helpers to add necessary adjustments
10910	* to their core operation.
10911	*
10912	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
10913	* any performance for debug and analysis features.
10914	*
10915	* @{
10916	*/
10917
10918	/**
10919	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
10920	* the debug run loop.
10921	*/
10922	typedef struct VMXRUNDBGSTATE
10923	{
10924	/** The RIP we started executing at. This is for detecting that we stepped. */
10925	uint64_t uRipStart;
10926	/** The CS we started executing with. */
10927	uint16_t uCsStart;
10928
10929	/** Whether we've actually modified the 1st execution control field. */
10930	bool fModifiedProcCtls : 1;
10931	/** Whether we've actually modified the 2nd execution control field. */
10932	bool fModifiedProcCtls2 : 1;
10933	/** Whether we've actually modified the exception bitmap. */
10934	bool fModifiedXcptBitmap : 1;
10935
10936	/** We desire the modified the CR0 mask to be cleared. */
10937	bool fClearCr0Mask : 1;
10938	/** We desire the modified the CR4 mask to be cleared. */
10939	bool fClearCr4Mask : 1;
10940	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
10941	uint32_t fCpe1Extra;
10942	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
10943	uint32_t fCpe1Unwanted;
10944	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
10945	uint32_t fCpe2Extra;
10946	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
10947	uint32_t bmXcptExtra;
10948	/** The sequence number of the Dtrace provider settings the state was
10949	* configured against. */
10950	uint32_t uDtraceSettingsSeqNo;
10951	/** VM-exits to check (one bit per VM-exit). */
10952	uint32_t bmExitsToCheck[3];
10953
10954	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
10955	uint32_t fProcCtlsInitial;
10956	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
10957	uint32_t fProcCtls2Initial;
10958	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
10959	uint32_t bmXcptInitial;
10960	} VMXRUNDBGSTATE;
10961	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
10962	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
10963
10964
10965	/**
10966	* Initializes the VMXRUNDBGSTATE structure.
10967	*
10968	* @param pVCpu The cross context virtual CPU structure of the
10969	* calling EMT.
10970	* @param pVmxTransient The VMX-transient structure.
10971	* @param pDbgState The debug state to initialize.
10972	*/
10973	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
10974	{
10975	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
10976	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
10977
10978	pDbgState->fModifiedProcCtls = false;
10979	pDbgState->fModifiedProcCtls2 = false;
10980	pDbgState->fModifiedXcptBitmap = false;
10981	pDbgState->fClearCr0Mask = false;
10982	pDbgState->fClearCr4Mask = false;
10983	pDbgState->fCpe1Extra = 0;
10984	pDbgState->fCpe1Unwanted = 0;
10985	pDbgState->fCpe2Extra = 0;
10986	pDbgState->bmXcptExtra = 0;
10987	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
10988	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
10989	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
10990	}
10991
10992
10993	/**
10994	* Updates the VMSC fields with changes requested by @a pDbgState.
10995	*
10996	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
10997	* immediately before executing guest code, i.e. when interrupts are disabled.
10998	* We don't check status codes here as we cannot easily assert or return in the
10999	* latter case.
11000	*
11001	* @param pVCpu The cross context virtual CPU structure.
11002	* @param pVmxTransient The VMX-transient structure.
11003	* @param pDbgState The debug state.
11004	*/
11005	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11006	{
11007	/*
11008	* Ensure desired flags in VMCS control fields are set.
11009	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11010	*
11011	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11012	* there should be no stale data in pCtx at this point.
11013	*/
11014	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11015	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11016	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11017	{
11018	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11019	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11020	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11021	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11022	pDbgState->fModifiedProcCtls = true;
11023	}
11024
11025	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11026	{
11027	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11028	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11029	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11030	pDbgState->fModifiedProcCtls2 = true;
11031	}
11032
11033	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11034	{
11035	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11036	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11037	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11038	pDbgState->fModifiedXcptBitmap = true;
11039	}
11040
11041	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11042	{
11043	pVmcsInfo->u64Cr0Mask = 0;
11044	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
11045	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11046	}
11047
11048	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11049	{
11050	pVmcsInfo->u64Cr4Mask = 0;
11051	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
11052	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11053	}
11054
11055	NOREF(pVCpu);
11056	}
11057
11058
11059	/**
11060	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11061	* re-entry next time around.
11062	*
11063	* @returns Strict VBox status code (i.e. informational status codes too).
11064	* @param pVCpu The cross context virtual CPU structure.
11065	* @param pVmxTransient The VMX-transient structure.
11066	* @param pDbgState The debug state.
11067	* @param rcStrict The return code from executing the guest using single
11068	* stepping.
11069	*/
11070	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11071	VBOXSTRICTRC rcStrict)
11072	{
11073	/*
11074	* Restore VM-exit control settings as we may not reenter this function the
11075	* next time around.
11076	*/
11077	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11078
11079	/* We reload the initial value, trigger what we can of recalculations the
11080	next time around. From the looks of things, that's all that's required atm. */
11081	if (pDbgState->fModifiedProcCtls)
11082	{
11083	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11084	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11085	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11086	AssertRCReturn(rc2, rc2);
11087	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11088	}
11089
11090	/* We're currently the only ones messing with this one, so just restore the
11091	cached value and reload the field. */
11092	if ( pDbgState->fModifiedProcCtls2
11093	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11094	{
11095	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11096	AssertRCReturn(rc2, rc2);
11097	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11098	}
11099
11100	/* If we've modified the exception bitmap, we restore it and trigger
11101	reloading and partial recalculation the next time around. */
11102	if (pDbgState->fModifiedXcptBitmap)
11103	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11104
11105	return rcStrict;
11106	}
11107
11108
11109	/**
11110	* Configures VM-exit controls for current DBGF and DTrace settings.
11111	*
11112	* This updates @a pDbgState and the VMCS execution control fields to reflect
11113	* the necessary VM-exits demanded by DBGF and DTrace.
11114	*
11115	* @param pVCpu The cross context virtual CPU structure.
11116	* @param pVmxTransient The VMX-transient structure. May update
11117	* fUpdatedTscOffsettingAndPreemptTimer.
11118	* @param pDbgState The debug state.
11119	*/
11120	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11121	{
11122	/*
11123	* Take down the dtrace serial number so we can spot changes.
11124	*/
11125	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11126	ASMCompilerBarrier();
11127
11128	/*
11129	* We'll rebuild most of the middle block of data members (holding the
11130	* current settings) as we go along here, so start by clearing it all.
11131	*/
11132	pDbgState->bmXcptExtra = 0;
11133	pDbgState->fCpe1Extra = 0;
11134	pDbgState->fCpe1Unwanted = 0;
11135	pDbgState->fCpe2Extra = 0;
11136	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11137	pDbgState->bmExitsToCheck[i] = 0;
11138
11139	/*
11140	* Software interrupts (INT XXh) - no idea how to trigger these...
11141	*/
11142	PVM pVM = pVCpu->CTX_SUFF(pVM);
11143	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11144	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11145	{
11146	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11147	}
11148
11149	/*
11150	* INT3 breakpoints - triggered by #BP exceptions.
11151	*/
11152	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11153	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11154
11155	/*
11156	* Exception bitmap and XCPT events+probes.
11157	*/
11158	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11159	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11160	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11161
11162	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11163	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11164	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11165	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11166	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11167	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11168	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11169	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11170	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11171	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11172	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11173	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11174	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11175	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11176	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11177	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11178	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11179	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11180
11181	if (pDbgState->bmXcptExtra)
11182	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11183
11184	/*
11185	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11186	*
11187	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11188	* So, when adding/changing/removing please don't forget to update it.
11189	*
11190	* Some of the macros are picking up local variables to save horizontal space,
11191	* (being able to see it in a table is the lesser evil here).
11192	*/
11193	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11194	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11195	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11196	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11197	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11198	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11199	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11200	} else do { } while (0)
11201	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11202	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11203	{ \
11204	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11205	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11206	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11207	} else do { } while (0)
11208	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11209	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11210	{ \
11211	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11212	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11213	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11214	} else do { } while (0)
11215	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11216	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11217	{ \
11218	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11219	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11220	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11221	} else do { } while (0)
11222
11223	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11224	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11225	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11226	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11227	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11228
11229	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11230	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11231	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11232	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11233	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11234	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11235	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11236	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11237	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11238	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11239	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11240	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11241	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11242	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11243	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11244	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11245	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11246	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11247	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11248	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11249	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11250	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11251	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11252	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11253	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11254	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11255	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11256	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11257	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11258	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11259	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11260	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11261	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11262	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11263	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11264	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11265
11266	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11267	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11268	{
11269	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11270	\| CPUMCTX_EXTRN_APIC_TPR);
11271	AssertRC(rc);
11272
11273	#if 0 /** @todo fix me */
11274	pDbgState->fClearCr0Mask = true;
11275	pDbgState->fClearCr4Mask = true;
11276	#endif
11277	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11278	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11279	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11280	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11281	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11282	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11283	require clearing here and in the loop if we start using it. */
11284	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11285	}
11286	else
11287	{
11288	if (pDbgState->fClearCr0Mask)
11289	{
11290	pDbgState->fClearCr0Mask = false;
11291	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11292	}
11293	if (pDbgState->fClearCr4Mask)
11294	{
11295	pDbgState->fClearCr4Mask = false;
11296	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11297	}
11298	}
11299	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11300	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11301
11302	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11303	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11304	{
11305	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11306	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11307	}
11308	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11309	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11310
11311	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11312	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11313	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11314	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11315	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11316	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11317	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11318	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11319	#if 0 /** @todo too slow, fix handler. */
11320	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11321	#endif
11322	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11323
11324	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11325	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11326	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11327	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11328	{
11329	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11330	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11331	}
11332	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11333	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11334	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11335	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11336
11337	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11338	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11339	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11340	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11341	{
11342	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11343	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11344	}
11345	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11346	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11347	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11348	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11349
11350	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11351	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11352	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11353	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11354	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11355	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11356	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11357	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11358	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11359	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11360	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11361	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11362	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11363	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11364	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11365	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11366	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11367	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11368	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11369	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11370	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11371	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11372
11373	#undef IS_EITHER_ENABLED
11374	#undef SET_ONLY_XBM_IF_EITHER_EN
11375	#undef SET_CPE1_XBM_IF_EITHER_EN
11376	#undef SET_CPEU_XBM_IF_EITHER_EN
11377	#undef SET_CPE2_XBM_IF_EITHER_EN
11378
11379	/*
11380	* Sanitize the control stuff.
11381	*/
11382	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11383	if (pDbgState->fCpe2Extra)
11384	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11385	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11386	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11387	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11388	{
11389	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11390	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11391	}
11392
11393	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11394	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11395	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11396	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11397	}
11398
11399
11400	/**
11401	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11402	* appropriate.
11403	*
11404	* The caller has checked the VM-exit against the
11405	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11406	* already, so we don't have to do that either.
11407	*
11408	* @returns Strict VBox status code (i.e. informational status codes too).
11409	* @param pVCpu The cross context virtual CPU structure.
11410	* @param pVmxTransient The VMX-transient structure.
11411	* @param uExitReason The VM-exit reason.
11412	*
11413	* @remarks The name of this function is displayed by dtrace, so keep it short
11414	* and to the point. No longer than 33 chars long, please.
11415	*/
11416	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11417	{
11418	/*
11419	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11420	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11421	*
11422	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11423	* does. Must add/change/remove both places. Same ordering, please.
11424	*
11425	* Added/removed events must also be reflected in the next section
11426	* where we dispatch dtrace events.
11427	*/
11428	bool fDtrace1 = false;
11429	bool fDtrace2 = false;
11430	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11431	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11432	uint32_t uEventArg = 0;
11433	#define SET_EXIT(a_EventSubName) \
11434	do { \
11435	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11436	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11437	} while (0)
11438	#define SET_BOTH(a_EventSubName) \
11439	do { \
11440	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11441	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11442	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11443	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11444	} while (0)
11445	switch (uExitReason)
11446	{
11447	case VMX_EXIT_MTF:
11448	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11449
11450	case VMX_EXIT_XCPT_OR_NMI:
11451	{
11452	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11453	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11454	{
11455	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11456	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11457	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11458	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11459	{
11460	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11461	{
11462	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11463	uEventArg = pVmxTransient->uExitIntErrorCode;
11464	}
11465	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11466	switch (enmEvent1)
11467	{
11468	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11469	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11470	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11471	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11472	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11473	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11474	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11475	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11476	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11477	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11478	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11479	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11480	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11481	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11482	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11483	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11484	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11485	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11486	default: break;
11487	}
11488	}
11489	else
11490	AssertFailed();
11491	break;
11492
11493	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11494	uEventArg = idxVector;
11495	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11496	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11497	break;
11498	}
11499	break;
11500	}
11501
11502	case VMX_EXIT_TRIPLE_FAULT:
11503	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11504	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11505	break;
11506	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11507	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11508	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11509	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11510	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11511
11512	/* Instruction specific VM-exits: */
11513	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11514	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11515	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11516	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11517	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11518	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11519	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11520	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11521	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11522	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11523	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11524	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11525	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11526	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11527	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11528	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11529	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11530	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11531	case VMX_EXIT_MOV_CRX:
11532	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11533	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11534	SET_BOTH(CRX_READ);
11535	else
11536	SET_BOTH(CRX_WRITE);
11537	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11538	break;
11539	case VMX_EXIT_MOV_DRX:
11540	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11541	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11542	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11543	SET_BOTH(DRX_READ);
11544	else
11545	SET_BOTH(DRX_WRITE);
11546	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11547	break;
11548	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11549	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11550	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11551	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11552	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11553	case VMX_EXIT_GDTR_IDTR_ACCESS:
11554	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11555	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11556	{
11557	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11558	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11559	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11560	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11561	}
11562	break;
11563
11564	case VMX_EXIT_LDTR_TR_ACCESS:
11565	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11566	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11567	{
11568	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11569	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11570	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11571	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11572	}
11573	break;
11574
11575	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11576	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11577	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11578	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11579	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11580	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11581	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11582	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11583	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11584	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11585	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11586
11587	/* Events that aren't relevant at this point. */
11588	case VMX_EXIT_EXT_INT:
11589	case VMX_EXIT_INT_WINDOW:
11590	case VMX_EXIT_NMI_WINDOW:
11591	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11592	case VMX_EXIT_PREEMPT_TIMER:
11593	case VMX_EXIT_IO_INSTR:
11594	break;
11595
11596	/* Errors and unexpected events. */
11597	case VMX_EXIT_INIT_SIGNAL:
11598	case VMX_EXIT_SIPI:
11599	case VMX_EXIT_IO_SMI:
11600	case VMX_EXIT_SMI:
11601	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11602	case VMX_EXIT_ERR_MSR_LOAD:
11603	case VMX_EXIT_ERR_MACHINE_CHECK:
11604	break;
11605
11606	default:
11607	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11608	break;
11609	}
11610	#undef SET_BOTH
11611	#undef SET_EXIT
11612
11613	/*
11614	* Dtrace tracepoints go first. We do them here at once so we don't
11615	* have to copy the guest state saving and stuff a few dozen times.
11616	* Down side is that we've got to repeat the switch, though this time
11617	* we use enmEvent since the probes are a subset of what DBGF does.
11618	*/
11619	if (fDtrace1 \|\| fDtrace2)
11620	{
11621	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11622	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11623	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11624	switch (enmEvent1)
11625	{
11626	/** @todo consider which extra parameters would be helpful for each probe. */
11627	case DBGFEVENT_END: break;
11628	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11629	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11630	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11631	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11632	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11633	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11634	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11635	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11636	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
11637	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
11638	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
11639	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
11640	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
11641	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
11642	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
11643	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
11644	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
11645	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
11646	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11647	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11648	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
11649	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
11650	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
11651	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
11652	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
11653	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
11654	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
11655	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11656	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11657	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11658	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11659	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11660	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
11661	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11662	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
11663	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
11664	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
11665	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
11666	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
11667	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
11668	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
11669	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
11670	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
11671	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
11672	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
11673	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
11674	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
11675	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
11676	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
11677	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
11678	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
11679	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
11680	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
11681	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
11682	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
11683	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
11684	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
11685	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
11686	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
11687	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
11688	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
11689	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
11690	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
11691	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
11692	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
11693	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
11694	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
11695	}
11696	switch (enmEvent2)
11697	{
11698	/** @todo consider which extra parameters would be helpful for each probe. */
11699	case DBGFEVENT_END: break;
11700	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
11701	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11702	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
11703	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
11704	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
11705	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
11706	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
11707	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
11708	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
11709	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11710	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11711	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11712	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11713	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11714	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
11715	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11716	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
11717	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
11718	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
11719	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
11720	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
11721	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
11722	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
11723	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
11724	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
11725	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
11726	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
11727	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
11728	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
11729	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
11730	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
11731	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
11732	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
11733	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
11734	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
11735	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
11736	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
11737	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
11738	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
11739	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
11740	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
11741	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
11742	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
11743	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
11744	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
11745	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
11746	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
11747	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
11748	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
11749	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
11750	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
11751	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
11752	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
11753	}
11754	}
11755
11756	/*
11757	* Fire of the DBGF event, if enabled (our check here is just a quick one,
11758	* the DBGF call will do a full check).
11759	*
11760	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
11761	* Note! If we have to events, we prioritize the first, i.e. the instruction
11762	* one, in order to avoid event nesting.
11763	*/
11764	PVM pVM = pVCpu->CTX_SUFF(pVM);
11765	if ( enmEvent1 != DBGFEVENT_END
11766	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
11767	{
11768	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11769	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
11770	if (rcStrict != VINF_SUCCESS)
11771	return rcStrict;
11772	}
11773	else if ( enmEvent2 != DBGFEVENT_END
11774	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
11775	{
11776	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11777	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
11778	if (rcStrict != VINF_SUCCESS)
11779	return rcStrict;
11780	}
11781
11782	return VINF_SUCCESS;
11783	}
11784
11785
11786	/**
11787	* Single-stepping VM-exit filtering.
11788	*
11789	* This is preprocessing the VM-exits and deciding whether we've gotten far
11790	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
11791	* handling is performed.
11792	*
11793	* @returns Strict VBox status code (i.e. informational status codes too).
11794	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
11795	* @param pVmxTransient The VMX-transient structure.
11796	* @param pDbgState The debug state.
11797	*/
11798	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11799	{
11800	/*
11801	* Expensive (saves context) generic dtrace VM-exit probe.
11802	*/
11803	uint32_t const uExitReason = pVmxTransient->uExitReason;
11804	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
11805	{ /* more likely */ }
11806	else
11807	{
11808	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11809	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11810	AssertRC(rc);
11811	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
11812	}
11813
11814	/*
11815	* Check for host NMI, just to get that out of the way.
11816	*/
11817	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
11818	{ /* normally likely */ }
11819	else
11820	{
11821	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11822	AssertRCReturn(rc2, rc2);
11823	uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11824	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11825	return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
11826	}
11827
11828	/*
11829	* Check for single stepping event if we're stepping.
11830	*/
11831	if (pVCpu->hm.s.fSingleInstruction)
11832	{
11833	switch (uExitReason)
11834	{
11835	case VMX_EXIT_MTF:
11836	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11837
11838	/* Various events: */
11839	case VMX_EXIT_XCPT_OR_NMI:
11840	case VMX_EXIT_EXT_INT:
11841	case VMX_EXIT_TRIPLE_FAULT:
11842	case VMX_EXIT_INT_WINDOW:
11843	case VMX_EXIT_NMI_WINDOW:
11844	case VMX_EXIT_TASK_SWITCH:
11845	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11846	case VMX_EXIT_APIC_ACCESS:
11847	case VMX_EXIT_EPT_VIOLATION:
11848	case VMX_EXIT_EPT_MISCONFIG:
11849	case VMX_EXIT_PREEMPT_TIMER:
11850
11851	/* Instruction specific VM-exits: */
11852	case VMX_EXIT_CPUID:
11853	case VMX_EXIT_GETSEC:
11854	case VMX_EXIT_HLT:
11855	case VMX_EXIT_INVD:
11856	case VMX_EXIT_INVLPG:
11857	case VMX_EXIT_RDPMC:
11858	case VMX_EXIT_RDTSC:
11859	case VMX_EXIT_RSM:
11860	case VMX_EXIT_VMCALL:
11861	case VMX_EXIT_VMCLEAR:
11862	case VMX_EXIT_VMLAUNCH:
11863	case VMX_EXIT_VMPTRLD:
11864	case VMX_EXIT_VMPTRST:
11865	case VMX_EXIT_VMREAD:
11866	case VMX_EXIT_VMRESUME:
11867	case VMX_EXIT_VMWRITE:
11868	case VMX_EXIT_VMXOFF:
11869	case VMX_EXIT_VMXON:
11870	case VMX_EXIT_MOV_CRX:
11871	case VMX_EXIT_MOV_DRX:
11872	case VMX_EXIT_IO_INSTR:
11873	case VMX_EXIT_RDMSR:
11874	case VMX_EXIT_WRMSR:
11875	case VMX_EXIT_MWAIT:
11876	case VMX_EXIT_MONITOR:
11877	case VMX_EXIT_PAUSE:
11878	case VMX_EXIT_GDTR_IDTR_ACCESS:
11879	case VMX_EXIT_LDTR_TR_ACCESS:
11880	case VMX_EXIT_INVEPT:
11881	case VMX_EXIT_RDTSCP:
11882	case VMX_EXIT_INVVPID:
11883	case VMX_EXIT_WBINVD:
11884	case VMX_EXIT_XSETBV:
11885	case VMX_EXIT_RDRAND:
11886	case VMX_EXIT_INVPCID:
11887	case VMX_EXIT_VMFUNC:
11888	case VMX_EXIT_RDSEED:
11889	case VMX_EXIT_XSAVES:
11890	case VMX_EXIT_XRSTORS:
11891	{
11892	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11893	AssertRCReturn(rc, rc);
11894	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
11895	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
11896	return VINF_EM_DBG_STEPPED;
11897	break;
11898	}
11899
11900	/* Errors and unexpected events: */
11901	case VMX_EXIT_INIT_SIGNAL:
11902	case VMX_EXIT_SIPI:
11903	case VMX_EXIT_IO_SMI:
11904	case VMX_EXIT_SMI:
11905	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11906	case VMX_EXIT_ERR_MSR_LOAD:
11907	case VMX_EXIT_ERR_MACHINE_CHECK:
11908	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
11909	break;
11910
11911	default:
11912	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11913	break;
11914	}
11915	}
11916
11917	/*
11918	* Check for debugger event breakpoints and dtrace probes.
11919	*/
11920	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
11921	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
11922	{
11923	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
11924	if (rcStrict != VINF_SUCCESS)
11925	return rcStrict;
11926	}
11927
11928	/*
11929	* Normal processing.
11930	*/
11931	#ifdef HMVMX_USE_FUNCTION_TABLE
11932	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
11933	#else
11934	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
11935	#endif
11936	}
11937
11938
11939	/**
11940	* Single steps guest code using hardware-assisted VMX.
11941	*
11942	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
11943	* but single-stepping through the hypervisor debugger.
11944	*
11945	* @returns Strict VBox status code (i.e. informational status codes too).
11946	* @param pVCpu The cross context virtual CPU structure.
11947	* @param pcLoops Pointer to the number of executed loops.
11948	*
11949	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
11950	*/
11951	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
11952	{
11953	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11954	Assert(pcLoops);
11955	Assert(*pcLoops <= cMaxResumeLoops);
11956
11957	VMXTRANSIENT VmxTransient;
11958	RT_ZERO(VmxTransient);
11959	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11960
11961	/* Set HMCPU indicators. */
11962	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
11963	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
11964	pVCpu->hm.s.fDebugWantRdTscExit = false;
11965	pVCpu->hm.s.fUsingDebugLoop = true;
11966
11967	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
11968	VMXRUNDBGSTATE DbgState;
11969	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
11970	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
11971
11972	/*
11973	* The loop.
11974	*/
11975	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11976	for (;;)
11977	{
11978	Assert(!HMR0SuspendPending());
11979	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11980	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11981	bool fStepping = pVCpu->hm.s.fSingleInstruction;
11982
11983	/* Set up VM-execution controls the next two can respond to. */
11984	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
11985
11986	/*
11987	* Preparatory work for running guest code, this may force us to
11988	* return to ring-3.
11989	*
11990	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11991	*/
11992	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
11993	if (rcStrict != VINF_SUCCESS)
11994	break;
11995
11996	/* Interrupts are disabled at this point! */
11997	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11998
11999	/* Override any obnoxious code in the above two calls. */
12000	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12001
12002	/*
12003	* Finally execute the guest.
12004	*/
12005	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12006
12007	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12008	/* Interrupts are re-enabled at this point! */
12009
12010	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12011	if (RT_SUCCESS(rcRun))
12012	{ /* very likely */ }
12013	else
12014	{
12015	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12016	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12017	return rcRun;
12018	}
12019
12020	/* Profile the VM-exit. */
12021	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12022	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12023	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12024	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12025	HMVMX_START_EXIT_DISPATCH_PROF();
12026
12027	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12028
12029	/*
12030	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12031	*/
12032	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12033	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12034	if (rcStrict != VINF_SUCCESS)
12035	break;
12036	if (++(*pcLoops) > cMaxResumeLoops)
12037	{
12038	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12039	rcStrict = VINF_EM_RAW_INTERRUPT;
12040	break;
12041	}
12042
12043	/*
12044	* Stepping: Did the RIP change, if so, consider it a single step.
12045	* Otherwise, make sure one of the TFs gets set.
12046	*/
12047	if (fStepping)
12048	{
12049	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12050	AssertRC(rc);
12051	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12052	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12053	{
12054	rcStrict = VINF_EM_DBG_STEPPED;
12055	break;
12056	}
12057	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12058	}
12059
12060	/*
12061	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12062	*/
12063	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12064	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12065	}
12066
12067	/*
12068	* Clear the X86_EFL_TF if necessary.
12069	*/
12070	if (pVCpu->hm.s.fClearTrapFlag)
12071	{
12072	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12073	AssertRC(rc);
12074	pVCpu->hm.s.fClearTrapFlag = false;
12075	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12076	}
12077	/** @todo there seems to be issues with the resume flag when the monitor trap
12078	* flag is pending without being used. Seen early in bios init when
12079	* accessing APIC page in protected mode. */
12080
12081	/*
12082	* Restore VM-exit control settings as we may not re-enter this function the
12083	* next time around.
12084	*/
12085	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12086
12087	/* Restore HMCPU indicators. */
12088	pVCpu->hm.s.fUsingDebugLoop = false;
12089	pVCpu->hm.s.fDebugWantRdTscExit = false;
12090	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12091
12092	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12093	return rcStrict;
12094	}
12095
12096
12097	/** @} */
12098
12099
12100	/**
12101	* Checks if any expensive dtrace probes are enabled and we should go to the
12102	* debug loop.
12103	*
12104	* @returns true if we should use debug loop, false if not.
12105	*/
12106	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12107	{
12108	/* It's probably faster to OR the raw 32-bit counter variables together.
12109	Since the variables are in an array and the probes are next to one
12110	another (more or less), we have good locality. So, better read
12111	eight-nine cache lines ever time and only have one conditional, than
12112	128+ conditionals, right? */
12113	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12114	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12115	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12116	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12117	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12118	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12119	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12120	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12121	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12122	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12123	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12124	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12125	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12126	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12127	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12128	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12129	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12130	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12131	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12132	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12133	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12134	) != 0
12135	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12136	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12137	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12138	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12139	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12140	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12141	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12142	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12143	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12144	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12145	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12146	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12147	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12148	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12149	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12150	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12151	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12152	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12153	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12154	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12155	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12156	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12157	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12158	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12159	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12160	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12161	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12162	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12163	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12164	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12165	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12166	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12167	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12168	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12169	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12170	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12171	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12172	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12173	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12174	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12175	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12176	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12177	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12178	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12179	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12180	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12181	) != 0
12182	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12183	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12184	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12185	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12186	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12187	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12188	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12189	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12190	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12191	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12192	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12193	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12194	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12195	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12196	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12197	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12198	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12199	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12200	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12201	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12202	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12203	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12204	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12205	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12206	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12207	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12208	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12209	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12210	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12211	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12212	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12213	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12214	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12215	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12216	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12217	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12218	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12219	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12220	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12221	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12222	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12223	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12224	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12225	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12226	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12227	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12228	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12229	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12230	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12231	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12232	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12233	) != 0;
12234	}
12235
12236
12237	/**
12238	* Runs the guest using hardware-assisted VMX.
12239	*
12240	* @returns Strict VBox status code (i.e. informational status codes too).
12241	* @param pVCpu The cross context virtual CPU structure.
12242	*/
12243	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
12244	{
12245	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12246	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12247	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12248	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12249
12250	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
12251
12252	VBOXSTRICTRC rcStrict;
12253	uint32_t cLoops = 0;
12254	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12255	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12256	#else
12257	bool const fInNestedGuestMode = false;
12258	#endif
12259	if (!fInNestedGuestMode)
12260	{
12261	if ( !pVCpu->hm.s.fUseDebugLoop
12262	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12263	&& !DBGFIsStepping(pVCpu)
12264	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12265	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12266	else
12267	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12268	}
12269	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12270	else
12271	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
12272
12273	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12274	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12275	#endif
12276
12277	if (rcStrict == VERR_EM_INTERPRETER)
12278	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12279	else if (rcStrict == VINF_EM_RESET)
12280	rcStrict = VINF_EM_TRIPLE_FAULT;
12281
12282	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12283	if (RT_FAILURE(rc2))
12284	{
12285	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12286	rcStrict = rc2;
12287	}
12288	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12289	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12290	return rcStrict;
12291	}
12292
12293
12294	#ifndef HMVMX_USE_FUNCTION_TABLE
12295	/**
12296	* Handles a guest VM-exit from hardware-assisted VMX execution.
12297	*
12298	* @returns Strict VBox status code (i.e. informational status codes too).
12299	* @param pVCpu The cross context virtual CPU structure.
12300	* @param pVmxTransient The VMX-transient structure.
12301	*/
12302	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12303	{
12304	#ifdef DEBUG_ramshankar
12305	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12306	do { \
12307	if (a_fSave != 0) \
12308	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
12309	VBOXSTRICTRC rcStrict = a_CallExpr; \
12310	if (a_fSave != 0) \
12311	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12312	return rcStrict; \
12313	} while (0)
12314	#else
12315	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12316	#endif
12317	uint32_t const rcReason = pVmxTransient->uExitReason;
12318	switch (rcReason)
12319	{
12320	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12321	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12322	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12323	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12324	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12325	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12326	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12327	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12328	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12329	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12330	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12331	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12332	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12333	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12334	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12335	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12336	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12337	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12338	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12339	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12340	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12341	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12342	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12343	case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
12344	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12345	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12346	case VMX_EXIT_GDTR_IDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12347	case VMX_EXIT_LDTR_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12348	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12349	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12350	case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
12351	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12352	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12353	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12354	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12355	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12356	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12357	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12358	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12359	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12360	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12361	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12362	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12363	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12364	#else
12365	case VMX_EXIT_VMCLEAR:
12366	case VMX_EXIT_VMLAUNCH:
12367	case VMX_EXIT_VMPTRLD:
12368	case VMX_EXIT_VMPTRST:
12369	case VMX_EXIT_VMREAD:
12370	case VMX_EXIT_VMRESUME:
12371	case VMX_EXIT_VMWRITE:
12372	case VMX_EXIT_VMXOFF:
12373	case VMX_EXIT_VMXON:
12374	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12375	#endif
12376
12377	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12378	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12379	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
12380	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
12381	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
12382	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
12383	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
12384	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12385	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
12386
12387	case VMX_EXIT_INVEPT:
12388	case VMX_EXIT_INVVPID:
12389	case VMX_EXIT_VMFUNC:
12390	case VMX_EXIT_XSAVES:
12391	case VMX_EXIT_XRSTORS:
12392	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12393
12394	case VMX_EXIT_ENCLS:
12395	case VMX_EXIT_RDSEED:
12396	case VMX_EXIT_PML_FULL:
12397	default:
12398	return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
12399	}
12400	#undef VMEXIT_CALL_RET
12401	}
12402	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12403
12404
12405	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12406	/**
12407	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12408	*
12409	* @returns Strict VBox status code (i.e. informational status codes too).
12410	* @param pVCpu The cross context virtual CPU structure.
12411	* @param pVmxTransient The VMX-transient structure.
12412	*/
12413	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12414	{
12415	uint32_t const rcReason = pVmxTransient->uExitReason;
12416	switch (rcReason)
12417	{
12418	case VMX_EXIT_EPT_MISCONFIG:
12419	case VMX_EXIT_EPT_VIOLATION:
12420	case VMX_EXIT_IO_INSTR:
12421	case VMX_EXIT_CPUID:
12422	case VMX_EXIT_RDTSC:
12423	case VMX_EXIT_RDTSCP:
12424	case VMX_EXIT_APIC_ACCESS:
12425	case VMX_EXIT_XCPT_OR_NMI:
12426	case VMX_EXIT_MOV_CRX:
12427	case VMX_EXIT_EXT_INT:
12428	case VMX_EXIT_INT_WINDOW:
12429	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12430	case VMX_EXIT_MWAIT:
12431	case VMX_EXIT_MONITOR:
12432	case VMX_EXIT_TASK_SWITCH:
12433	case VMX_EXIT_PREEMPT_TIMER:
12434	case VMX_EXIT_RDMSR:
12435	case VMX_EXIT_WRMSR:
12436	case VMX_EXIT_VMCALL:
12437	case VMX_EXIT_MOV_DRX:
12438	case VMX_EXIT_HLT:
12439	case VMX_EXIT_INVD:
12440	case VMX_EXIT_INVLPG:
12441	case VMX_EXIT_RSM:
12442	case VMX_EXIT_MTF:
12443	case VMX_EXIT_PAUSE:
12444	case VMX_EXIT_GDTR_IDTR_ACCESS:
12445	case VMX_EXIT_LDTR_TR_ACCESS:
12446	case VMX_EXIT_WBINVD:
12447	case VMX_EXIT_XSETBV:
12448	case VMX_EXIT_RDRAND:
12449	case VMX_EXIT_INVPCID:
12450	case VMX_EXIT_GETSEC:
12451	case VMX_EXIT_RDPMC:
12452	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12453	case VMX_EXIT_VMCLEAR:
12454	case VMX_EXIT_VMLAUNCH:
12455	case VMX_EXIT_VMPTRLD:
12456	case VMX_EXIT_VMPTRST:
12457	case VMX_EXIT_VMREAD:
12458	case VMX_EXIT_VMRESUME:
12459	case VMX_EXIT_VMWRITE:
12460	case VMX_EXIT_VMXOFF:
12461	case VMX_EXIT_VMXON:
12462	#endif
12463	case VMX_EXIT_TRIPLE_FAULT:
12464	case VMX_EXIT_NMI_WINDOW:
12465	case VMX_EXIT_INIT_SIGNAL:
12466	case VMX_EXIT_SIPI:
12467	case VMX_EXIT_IO_SMI:
12468	case VMX_EXIT_SMI:
12469	case VMX_EXIT_ERR_MSR_LOAD:
12470	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12471	case VMX_EXIT_ERR_MACHINE_CHECK:
12472
12473	case VMX_EXIT_INVEPT:
12474	case VMX_EXIT_INVVPID:
12475	case VMX_EXIT_VMFUNC:
12476	case VMX_EXIT_XSAVES:
12477	case VMX_EXIT_XRSTORS:
12478
12479	case VMX_EXIT_ENCLS:
12480	case VMX_EXIT_RDSEED:
12481	case VMX_EXIT_PML_FULL:
12482	default:
12483	return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
12484	}
12485	#undef VMEXIT_CALL_RET
12486	}
12487	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12488
12489
12490	#ifdef VBOX_STRICT
12491	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
12492	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
12493	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
12494
12495	# define HMVMX_ASSERT_PREEMPT_CPUID() \
12496	do { \
12497	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
12498	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
12499	} while (0)
12500
12501	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12502	do { \
12503	AssertPtr((a_pVCpu)); \
12504	AssertPtr((a_pVmxTransient)); \
12505	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
12506	Assert((a_pVmxTransient)->pVmcsInfo); \
12507	Assert(ASMIntAreEnabled()); \
12508	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12509	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
12510	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
12511	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12512	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
12513	HMVMX_ASSERT_PREEMPT_CPUID(); \
12514	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12515	} while (0)
12516
12517	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12518	do { \
12519	Log4Func(("\n")); \
12520	} while (0)
12521	#else
12522	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12523	do { \
12524	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12525	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
12526	} while (0)
12527	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
12528	#endif
12529
12530
12531	/**
12532	* Advances the guest RIP by the specified number of bytes.
12533	*
12534	* @param pVCpu The cross context virtual CPU structure.
12535	* @param cbInstr Number of bytes to advance the RIP by.
12536	*
12537	* @remarks No-long-jump zone!!!
12538	*/
12539	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
12540	{
12541	/* Advance the RIP. */
12542	pVCpu->cpum.GstCtx.rip += cbInstr;
12543	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12544
12545	/* Update interrupt inhibition. */
12546	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
12547	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
12548	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12549	}
12550
12551
12552	/**
12553	* Advances the guest RIP after reading it from the VMCS.
12554	*
12555	* @returns VBox status code, no informational status codes.
12556	* @param pVCpu The cross context virtual CPU structure.
12557	* @param pVmxTransient The VMX-transient structure.
12558	*
12559	* @remarks No-long-jump zone!!!
12560	*/
12561	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12562	{
12563	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12564	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
12565	AssertRCReturn(rc, rc);
12566
12567	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
12568	return VINF_SUCCESS;
12569	}
12570
12571
12572	/**
12573	* Handle a condition that occurred while delivering an event through the guest
12574	* IDT.
12575	*
12576	* @returns Strict VBox status code (i.e. informational status codes too).
12577	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
12578	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
12579	* to continue execution of the guest which will delivery the \#DF.
12580	* @retval VINF_EM_RESET if we detected a triple-fault condition.
12581	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
12582	*
12583	* @param pVCpu The cross context virtual CPU structure.
12584	* @param pVmxTransient The VMX-transient structure.
12585	*
12586	* @remarks No-long-jump zone!!!
12587	*/
12588	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12589	{
12590	uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12591
12592	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12593	rc2 \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12594	AssertRCReturn(rc2, rc2);
12595
12596	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
12597	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12598	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12599	{
12600	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12601	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12602
12603	/*
12604	* If the event was a software interrupt (generated with INT n) or a software exception
12605	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
12606	* can handle the VM-exit and continue guest execution which will re-execute the
12607	* instruction rather than re-injecting the exception, as that can cause premature
12608	* trips to ring-3 before injection and involve TRPM which currently has no way of
12609	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
12610	* the problem).
12611	*/
12612	IEMXCPTRAISE enmRaise;
12613	IEMXCPTRAISEINFO fRaiseInfo;
12614	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
12615	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
12616	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
12617	{
12618	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
12619	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12620	}
12621	else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
12622	{
12623	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12624	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
12625	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
12626	/** @todo Make AssertMsgReturn as just AssertMsg later. */
12627	AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
12628	("Unexpected VM-exit interruption vector type %#x!\n", uExitVectorType), VERR_VMX_IPE_5);
12629
12630	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
12631
12632	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
12633	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
12634	{
12635	pVmxTransient->fVectoringPF = true;
12636	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12637	}
12638	}
12639	else
12640	{
12641	/*
12642	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
12643	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
12644	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
12645	*/
12646	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12647	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12648	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
12649	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12650	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12651	}
12652
12653	/*
12654	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
12655	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
12656	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
12657	* subsequent VM-entry would fail.
12658	*
12659	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
12660	*/
12661	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS)
12662	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12663	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
12664	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
12665	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
12666	{
12667	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
12668	}
12669
12670	switch (enmRaise)
12671	{
12672	case IEMXCPTRAISE_CURRENT_XCPT:
12673	{
12674	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
12675	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
12676	Assert(rcStrict == VINF_SUCCESS);
12677	break;
12678	}
12679
12680	case IEMXCPTRAISE_PREV_EVENT:
12681	{
12682	uint32_t u32ErrCode;
12683	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
12684	{
12685	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12686	AssertRCReturn(rc2, rc2);
12687	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12688	}
12689	else
12690	u32ErrCode = 0;
12691
12692	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
12693	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
12694	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12695	0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
12696
12697	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
12698	pVCpu->hm.s.Event.u32ErrCode));
12699	Assert(rcStrict == VINF_SUCCESS);
12700	break;
12701	}
12702
12703	case IEMXCPTRAISE_REEXEC_INSTR:
12704	Assert(rcStrict == VINF_SUCCESS);
12705	break;
12706
12707	case IEMXCPTRAISE_DOUBLE_FAULT:
12708	{
12709	/*
12710	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
12711	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
12712	*/
12713	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
12714	{
12715	pVmxTransient->fVectoringDoublePF = true;
12716	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
12717	pVCpu->cpum.GstCtx.cr2));
12718	rcStrict = VINF_SUCCESS;
12719	}
12720	else
12721	{
12722	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
12723	hmR0VmxSetPendingXcptDF(pVCpu);
12724	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
12725	uIdtVector, uExitVector));
12726	rcStrict = VINF_HM_DOUBLE_FAULT;
12727	}
12728	break;
12729	}
12730
12731	case IEMXCPTRAISE_TRIPLE_FAULT:
12732	{
12733	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
12734	rcStrict = VINF_EM_RESET;
12735	break;
12736	}
12737
12738	case IEMXCPTRAISE_CPU_HANG:
12739	{
12740	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
12741	rcStrict = VERR_EM_GUEST_CPU_HANG;
12742	break;
12743	}
12744
12745	default:
12746	{
12747	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
12748	rcStrict = VERR_VMX_IPE_2;
12749	break;
12750	}
12751	}
12752	}
12753	else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
12754	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
12755	&& uExitVector != X86_XCPT_DF
12756	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
12757	{
12758	/*
12759	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
12760	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
12761	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
12762	*/
12763	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
12764	{
12765	Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
12766	VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
12767	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
12768	}
12769	}
12770
12771	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
12772	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
12773	return rcStrict;
12774	}
12775
12776
12777	/** @name VM-exit handlers.
12778	* @{
12779	*/
12780	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12781	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12782	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12783
12784	/**
12785	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
12786	*/
12787	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12788	{
12789	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12790	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
12791	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
12792	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
12793	return VINF_SUCCESS;
12794	return VINF_EM_RAW_INTERRUPT;
12795	}
12796
12797
12798	/**
12799	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
12800	*/
12801	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12802	{
12803	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12804	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
12805
12806	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12807	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12808	AssertRCReturn(rc, rc);
12809
12810	uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12811	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
12812	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
12813	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
12814
12815	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12816	{
12817	/*
12818	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
12819	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
12820	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
12821	*
12822	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
12823	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
12824	*/
12825	VMXDispatchHostNmi();
12826	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
12827	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12828	return VINF_SUCCESS;
12829	}
12830
12831	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12832	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12833	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
12834	{ /* likely */ }
12835	else
12836	{
12837	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
12838	rcStrictRc1 = VINF_SUCCESS;
12839	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12840	return rcStrictRc1;
12841	}
12842
12843	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
12844	uint32_t uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
12845	switch (uIntType)
12846	{
12847	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
12848	Assert(uVector == X86_XCPT_DB);
12849	RT_FALL_THRU();
12850	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
12851	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
12852	RT_FALL_THRU();
12853	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
12854	{
12855	/*
12856	* If there's any exception caused as a result of event injection, the resulting
12857	* secondary/final execption will be pending, we shall continue guest execution
12858	* after injecting the event. The page-fault case is complicated and we manually
12859	* handle any currently pending event in hmR0VmxExitXcptPF.
12860	*/
12861	if (!pVCpu->hm.s.Event.fPending)
12862	{ /* likely */ }
12863	else if (uVector != X86_XCPT_PF)
12864	{
12865	rc = VINF_SUCCESS;
12866	break;
12867	}
12868
12869	switch (uVector)
12870	{
12871	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
12872	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
12873	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
12874	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
12875	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
12876	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
12877
12878	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
12879	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12880	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
12881	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12882	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
12883	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12884	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
12885	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12886	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
12887	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12888	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
12889	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12890	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
12891	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12892	default:
12893	{
12894	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
12895	if (pVmcsInfo->RealMode.fRealOnV86Active)
12896	{
12897	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
12898	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
12899	Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
12900
12901	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
12902	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12903	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12904	AssertRCReturn(rc, rc);
12905	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
12906	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
12907	0 /* GCPtrFaultAddress */);
12908	}
12909	else
12910	{
12911	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
12912	pVCpu->hm.s.u32HMError = uVector;
12913	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
12914	}
12915	break;
12916	}
12917	}
12918	break;
12919	}
12920
12921	default:
12922	{
12923	pVCpu->hm.s.u32HMError = uExitIntInfo;
12924	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
12925	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
12926	break;
12927	}
12928	}
12929	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12930	return rc;
12931	}
12932
12933
12934	/**
12935	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
12936	*/
12937	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12938	{
12939	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12940
12941	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
12942	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12943	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
12944	AssertRCReturn(rc, rc);
12945
12946	/* Evaluate and deliver pending events and resume guest execution. */
12947	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
12948	return VINF_SUCCESS;
12949	}
12950
12951
12952	/**
12953	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
12954	*/
12955	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12956	{
12957	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12958
12959	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12960	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
12961	{
12962	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
12963	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12964	}
12965
12966	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS));
12967
12968	/*
12969	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
12970	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
12971	*/
12972	uint32_t fIntrState;
12973	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
12974	AssertRCReturn(rc, rc);
12975	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
12976	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
12977	{
12978	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
12979	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12980
12981	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
12982	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
12983	AssertRCReturn(rc, rc);
12984	}
12985
12986	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
12987	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
12988	AssertRCReturn(rc, rc);
12989
12990	/* Evaluate and deliver pending events and resume guest execution. */
12991	return VINF_SUCCESS;
12992	}
12993
12994
12995	/**
12996	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
12997	*/
12998	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12999	{
13000	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13001	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13002	}
13003
13004
13005	/**
13006	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
13007	*/
13008	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13009	{
13010	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13011	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13012	}
13013
13014
13015	/**
13016	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
13017	*/
13018	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13019	{
13020	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13021
13022	/*
13023	* Get the state we need and update the exit history entry.
13024	*/
13025	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13026	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13027	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13028	AssertRCReturn(rc, rc);
13029
13030	VBOXSTRICTRC rcStrict;
13031	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
13032	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
13033	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
13034	if (!pExitRec)
13035	{
13036	/*
13037	* Regular CPUID instruction execution.
13038	*/
13039	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
13040	if (rcStrict == VINF_SUCCESS)
13041	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13042	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13043	{
13044	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13045	rcStrict = VINF_SUCCESS;
13046	}
13047	}
13048	else
13049	{
13050	/*
13051	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
13052	*/
13053	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13054	AssertRCReturn(rc2, rc2);
13055
13056	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
13057	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
13058
13059	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
13060	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13061
13062	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
13063	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13064	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
13065	}
13066	return rcStrict;
13067	}
13068
13069
13070	/**
13071	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
13072	*/
13073	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13074	{
13075	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13076
13077	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13078	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
13079	AssertRCReturn(rc, rc);
13080
13081	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
13082	return VINF_EM_RAW_EMULATE_INSTR;
13083
13084	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
13085	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13086	}
13087
13088
13089	/**
13090	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
13091	*/
13092	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13093	{
13094	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13095
13096	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13097	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13098	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13099	AssertRCReturn(rc, rc);
13100
13101	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
13102	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13103	{
13104	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13105	we must reset offsetting on VM-entry. See @bugref{6634}. */
13106	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13107	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13108	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13109	}
13110	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13111	{
13112	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13113	rcStrict = VINF_SUCCESS;
13114	}
13115	return rcStrict;
13116	}
13117
13118
13119	/**
13120	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
13121	*/
13122	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13123	{
13124	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13125
13126	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13127	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
13128	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13129	AssertRCReturn(rc, rc);
13130
13131	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
13132	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13133	{
13134	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13135	we must reset offsetting on VM-reentry. See @bugref{6634}. */
13136	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13137	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13138	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13139	}
13140	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13141	{
13142	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13143	rcStrict = VINF_SUCCESS;
13144	}
13145	return rcStrict;
13146	}
13147
13148
13149	/**
13150	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
13151	*/
13152	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13153	{
13154	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13155
13156	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13157	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
13158	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13159	AssertRCReturn(rc, rc);
13160
13161	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13162	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13163	if (RT_LIKELY(rc == VINF_SUCCESS))
13164	{
13165	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13166	Assert(pVmxTransient->cbInstr == 2);
13167	}
13168	else
13169	{
13170	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
13171	rc = VERR_EM_INTERPRETER;
13172	}
13173	return rc;
13174	}
13175
13176
13177	/**
13178	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
13179	*/
13180	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13181	{
13182	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13183
13184	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
13185	if (EMAreHypercallInstructionsEnabled(pVCpu))
13186	{
13187	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13188	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
13189	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13190	AssertRCReturn(rc, rc);
13191
13192	/* Perform the hypercall. */
13193	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
13194	if (rcStrict == VINF_SUCCESS)
13195	{
13196	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13197	AssertRCReturn(rc, rc);
13198	}
13199	else
13200	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
13201	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
13202	\|\| RT_FAILURE(rcStrict));
13203
13204	/* If the hypercall changes anything other than guest's general-purpose registers,
13205	we would need to reload the guest changed bits here before VM-entry. */
13206	}
13207	else
13208	Log4Func(("Hypercalls not enabled\n"));
13209
13210	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
13211	if (RT_FAILURE(rcStrict))
13212	{
13213	hmR0VmxSetPendingXcptUD(pVCpu);
13214	rcStrict = VINF_SUCCESS;
13215	}
13216
13217	return rcStrict;
13218	}
13219
13220
13221	/**
13222	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
13223	*/
13224	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13225	{
13226	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13227	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
13228
13229	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13230	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13231	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13232	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13233	AssertRCReturn(rc, rc);
13234
13235	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
13236
13237	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
13238	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13239	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13240	{
13241	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13242	rcStrict = VINF_SUCCESS;
13243	}
13244	else
13245	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n", pVmxTransient->uExitQual,
13246	VBOXSTRICTRC_VAL(rcStrict)));
13247	return rcStrict;
13248	}
13249
13250
13251	/**
13252	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
13253	*/
13254	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13255	{
13256	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13257
13258	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13259	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13260	AssertRCReturn(rc, rc);
13261
13262	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13263	rc = EMInterpretMonitor(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13264	if (RT_LIKELY(rc == VINF_SUCCESS))
13265	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13266	else
13267	{
13268	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
13269	rc = VERR_EM_INTERPRETER;
13270	}
13271	return rc;
13272	}
13273
13274
13275	/**
13276	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
13277	*/
13278	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13279	{
13280	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13281
13282	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13283	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13284	AssertRCReturn(rc, rc);
13285
13286	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13287	VBOXSTRICTRC rc2 = EMInterpretMWait(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13288	rc = VBOXSTRICTRC_VAL(rc2);
13289	if (RT_LIKELY( rc == VINF_SUCCESS
13290	\|\| rc == VINF_EM_HALT))
13291	{
13292	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13293	AssertRCReturn(rc3, rc3);
13294
13295	if ( rc == VINF_EM_HALT
13296	&& EMMonitorWaitShouldContinue(pVCpu, pCtx))
13297	rc = VINF_SUCCESS;
13298	}
13299	else
13300	{
13301	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
13302	rc = VERR_EM_INTERPRETER;
13303	}
13304	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
13305	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
13306	return rc;
13307	}
13308
13309
13310	/**
13311	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
13312	*/
13313	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13314	{
13315	/*
13316	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
13317	* mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
13318	* treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
13319	* VMX root operation. If we get here, something funny is going on.
13320	*
13321	* See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
13322	*/
13323	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13324	AssertMsgFailed(("Unexpected RSM VM-exit\n"));
13325	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13326	}
13327
13328
13329	/**
13330	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
13331	*/
13332	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13333	{
13334	/*
13335	* This can only happen if we support dual-monitor treatment of SMI, which can be activated
13336	* by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
13337	* this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
13338	* an SMI. If we get here, something funny is going on.
13339	*
13340	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
13341	* See Intel spec. 25.3 "Other Causes of VM-Exits"
13342	*/
13343	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13344	AssertMsgFailed(("Unexpected SMI VM-exit\n"));
13345	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13346	}
13347
13348
13349	/**
13350	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
13351	*/
13352	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13353	{
13354	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
13355	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13356	AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
13357	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13358	}
13359
13360
13361	/**
13362	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
13363	*/
13364	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13365	{
13366	/*
13367	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
13368	* We don't make use of it as our guests don't have direct access to the host LAPIC.
13369	* See Intel spec. 25.3 "Other Causes of VM-exits".
13370	*/
13371	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13372	AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
13373	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13374	}
13375
13376
13377	/**
13378	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
13379	* VM-exit.
13380	*/
13381	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13382	{
13383	/*
13384	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
13385	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
13386	*
13387	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
13388	* See Intel spec. "23.8 Restrictions on VMX operation".
13389	*/
13390	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13391	return VINF_SUCCESS;
13392	}
13393
13394
13395	/**
13396	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
13397	* VM-exit.
13398	*/
13399	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13400	{
13401	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13402	return VINF_EM_RESET;
13403	}
13404
13405
13406	/**
13407	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
13408	*/
13409	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13410	{
13411	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13412
13413	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13414	AssertRCReturn(rc, rc);
13415
13416	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
13417	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
13418	rc = VINF_SUCCESS;
13419	else
13420	rc = VINF_EM_HALT;
13421
13422	if (rc != VINF_SUCCESS)
13423	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
13424	return rc;
13425	}
13426
13427
13428	/**
13429	* VM-exit handler for instructions that result in a \#UD exception delivered to
13430	* the guest.
13431	*/
13432	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13433	{
13434	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13435	hmR0VmxSetPendingXcptUD(pVCpu);
13436	return VINF_SUCCESS;
13437	}
13438
13439
13440	/**
13441	* VM-exit handler for expiry of the VMX-preemption timer.
13442	*/
13443	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13444	{
13445	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13446
13447	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
13448	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13449
13450	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
13451	PVM pVM = pVCpu->CTX_SUFF(pVM);
13452	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
13453	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
13454	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
13455	}
13456
13457
13458	/**
13459	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
13460	*/
13461	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13462	{
13463	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13464
13465	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13466	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13467	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
13468	AssertRCReturn(rc, rc);
13469
13470	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
13471	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13472	: HM_CHANGED_RAISED_XCPT_MASK);
13473
13474	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13475	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
13476
13477	return rcStrict;
13478	}
13479
13480
13481	/**
13482	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
13483	*/
13484	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13485	{
13486	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13487	/** @todo Use VM-exit instruction information. */
13488	return VERR_EM_INTERPRETER;
13489	}
13490
13491
13492	/**
13493	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
13494	* Error VM-exit.
13495	*/
13496	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13497	{
13498	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13499	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13500	AssertRCReturn(rc, rc);
13501
13502	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo);
13503	if (RT_FAILURE(rc))
13504	return rc;
13505
13506	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
13507	NOREF(uInvalidReason);
13508
13509	#ifdef VBOX_STRICT
13510	uint32_t fIntrState;
13511	RTHCUINTREG uHCReg;
13512	uint64_t u64Val;
13513	uint32_t u32Val;
13514	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
13515	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
13516	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
13517	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13518	AssertRCReturn(rc, rc);
13519
13520	Log4(("uInvalidReason %u\n", uInvalidReason));
13521	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
13522	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
13523	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
13524	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
13525
13526	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
13527	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
13528	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
13529	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
13530	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
13531	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13532	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
13533	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
13534	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
13535	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13536	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
13537	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
13538
13539	hmR0DumpRegs(pVCpu);
13540	#endif
13541
13542	return VERR_VMX_INVALID_GUEST_STATE;
13543	}
13544
13545
13546	/**
13547	* VM-exit handler for VM-entry failure due to an MSR-load
13548	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
13549	*/
13550	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13551	{
13552	AssertMsgFailed(("Unexpected MSR-load exit\n"));
13553	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13554	}
13555
13556
13557	/**
13558	* VM-exit handler for VM-entry failure due to a machine-check event
13559	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
13560	*/
13561	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13562	{
13563	AssertMsgFailed(("Unexpected machine-check event exit\n"));
13564	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13565	}
13566
13567
13568	/**
13569	* VM-exit handler for all undefined reasons. Should never ever happen.. in
13570	* theory.
13571	*/
13572	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13573	{
13574	RT_NOREF2(pVCpu, pVmxTransient);
13575	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
13576	return VERR_VMX_UNDEFINED_EXIT_CODE;
13577	}
13578
13579
13580	/**
13581	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
13582	* (VMX_EXIT_GDTR_IDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
13583	* Conditional VM-exit.
13584	*/
13585	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13586	{
13587	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13588
13589	/* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
13590	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
13591	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13592	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
13593	return VERR_EM_INTERPRETER;
13594	AssertMsgFailed(("Unexpected XDTR access\n"));
13595	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13596	}
13597
13598
13599	/**
13600	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
13601	*/
13602	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13603	{
13604	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13605
13606	/* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
13607	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13608	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
13609	return VERR_EM_INTERPRETER;
13610	AssertMsgFailed(("Unexpected RDRAND exit\n"));
13611	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13612	}
13613
13614
13615	/**
13616	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
13617	*/
13618	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13619	{
13620	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13621
13622	/** @todo Optimize this: We currently drag in in the whole MSR state
13623	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13624	* MSRs required. That would require changes to IEM and possibly CPUM too.
13625	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13626	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13627	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13628	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13629	switch (idMsr)
13630	{
13631	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13632	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13633	}
13634
13635	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13636	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13637	AssertRCReturn(rc, rc);
13638
13639	Log4Func(("ecx=%#RX32\n", idMsr));
13640
13641	#ifdef VBOX_STRICT
13642	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
13643	{
13644	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
13645	&& idMsr != MSR_K6_EFER)
13646	{
13647	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
13648	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13649	}
13650	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13651	{
13652	Assert(pVmcsInfo->pvMsrBitmap);
13653	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13654	if (fMsrpm & VMXMSRPM_ALLOW_RD)
13655	{
13656	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
13657	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13658	}
13659	}
13660	}
13661	#endif
13662
13663	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
13664	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
13665	if (rcStrict == VINF_SUCCESS)
13666	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13667	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
13668	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13669	{
13670	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13671	rcStrict = VINF_SUCCESS;
13672	}
13673	else
13674	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13675
13676	return rcStrict;
13677	}
13678
13679
13680	/**
13681	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
13682	*/
13683	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13684	{
13685	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13686
13687	/** @todo Optimize this: We currently drag in in the whole MSR state
13688	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13689	* MSRs required. That would require changes to IEM and possibly CPUM too.
13690	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13691	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13692	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13693
13694	/*
13695	* The FS and GS base MSRs are not part of the above all-MSRs mask.
13696	* Although we don't need to fetch the base as it will be overwritten shortly, while
13697	* loading guest-state we would also load the entire segment register including limit
13698	* and attributes and thus we need to load them here.
13699	*/
13700	switch (idMsr)
13701	{
13702	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13703	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13704	}
13705
13706	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13707	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13708	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13709	AssertRCReturn(rc, rc);
13710
13711	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
13712
13713	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
13714	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
13715
13716	if (rcStrict == VINF_SUCCESS)
13717	{
13718	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13719
13720	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
13721	if ( idMsr == MSR_IA32_APICBASE
13722	\|\| ( idMsr >= MSR_IA32_X2APIC_START
13723	&& idMsr <= MSR_IA32_X2APIC_END))
13724	{
13725	/*
13726	* We've already saved the APIC related guest-state (TPR) in post-run phase.
13727	* When full APIC register virtualization is implemented we'll have to make
13728	* sure APIC state is saved from the VMCS before IEM changes it.
13729	*/
13730	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
13731	}
13732	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
13733	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13734	else if (idMsr == MSR_K6_EFER)
13735	{
13736	/*
13737	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
13738	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
13739	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
13740	*/
13741	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
13742	}
13743
13744	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
13745	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
13746	{
13747	switch (idMsr)
13748	{
13749	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
13750	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
13751	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
13752	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
13753	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
13754	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
13755	default:
13756	{
13757	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
13758	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
13759	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13760	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
13761	break;
13762	}
13763	}
13764	}
13765	#ifdef VBOX_STRICT
13766	else
13767	{
13768	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
13769	switch (idMsr)
13770	{
13771	case MSR_IA32_SYSENTER_CS:
13772	case MSR_IA32_SYSENTER_EIP:
13773	case MSR_IA32_SYSENTER_ESP:
13774	case MSR_K8_FS_BASE:
13775	case MSR_K8_GS_BASE:
13776	{
13777	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13778	Assert(fMsrpm == (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR));
13779
13780	uint32_t u32Proc;
13781	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Proc);
13782	AssertRC(rc);
13783	Assert(u32Proc == pVmcsInfo->u32ProcCtls);
13784	Assert(u32Proc & VMX_PROC_CTLS_USE_MSR_BITMAPS);
13785
13786	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32 fMsrpm=%#RX32\n", idMsr, fMsrpm));
13787	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13788	}
13789
13790	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
13791	default:
13792	{
13793	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
13794	{
13795	/* EFER MSR writes are always intercepted. */
13796	if (idMsr != MSR_K6_EFER)
13797	{
13798	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
13799	idMsr));
13800	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13801	}
13802	}
13803
13804	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13805	{
13806	Assert(pVmcsInfo->pvMsrBitmap);
13807	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13808	if (fMsrpm & VMXMSRPM_ALLOW_WR)
13809	{
13810	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
13811	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13812	}
13813	}
13814	break;
13815	}
13816	}
13817	}
13818	#endif /* VBOX_STRICT */
13819	}
13820	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13821	{
13822	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13823	rcStrict = VINF_SUCCESS;
13824	}
13825	else
13826	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13827
13828	return rcStrict;
13829	}
13830
13831
13832	/**
13833	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
13834	*/
13835	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13836	{
13837	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13838	/** @todo The guest has likely hit a contended spinlock. We might want to
13839	* poke a schedule different guest VCPU. */
13840	return VINF_EM_RAW_INTERRUPT;
13841	}
13842
13843
13844	/**
13845	* VM-exit handler for when the TPR value is lowered below the specified
13846	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
13847	*/
13848	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13849	{
13850	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13851	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
13852
13853	/*
13854	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
13855	* We'll re-evaluate pending interrupts and inject them before the next VM
13856	* entry so we can just continue execution here.
13857	*/
13858	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
13859	return VINF_SUCCESS;
13860	}
13861
13862
13863	/**
13864	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
13865	* VM-exit.
13866	*
13867	* @retval VINF_SUCCESS when guest execution can continue.
13868	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
13869	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
13870	* interpreter.
13871	*/
13872	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13873	{
13874	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13875	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
13876
13877	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13878	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13879	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13880	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13881	AssertRCReturn(rc, rc);
13882
13883	VBOXSTRICTRC rcStrict;
13884	PVM pVM = pVCpu->CTX_SUFF(pVM);
13885	RTGCUINTPTR const uExitQual = pVmxTransient->uExitQual;
13886	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
13887	switch (uAccessType)
13888	{
13889	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
13890	{
13891	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
13892	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
13893	VMX_EXIT_QUAL_CRX_GENREG(uExitQual));
13894	AssertMsg( rcStrict == VINF_SUCCESS
13895	\|\| rcStrict == VINF_IEM_RAISED_XCPT
13896	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13897
13898	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
13899	{
13900	case 0:
13901	{
13902	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13903	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13904	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
13905	Log4Func(("CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
13906
13907	/*
13908	* This is a kludge for handling switches back to real mode when we try to use
13909	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
13910	* deal with special selector values, so we have to return to ring-3 and run
13911	* there till the selector values are V86 mode compatible.
13912	*
13913	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
13914	* latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
13915	* at the end of this function.
13916	*/
13917	if ( rc == VINF_SUCCESS
13918	&& !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
13919	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
13920	&& (uOldCr0 & X86_CR0_PE)
13921	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
13922	{
13923	/** @todo check selectors rather than returning all the time. */
13924	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
13925	rcStrict = VINF_EM_RESCHEDULE_REM;
13926	}
13927	break;
13928	}
13929
13930	case 2:
13931	{
13932	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
13933	/* Nothing to do here, CR2 it's not part of the VMCS. */
13934	break;
13935	}
13936
13937	case 3:
13938	{
13939	Assert( !pVM->hm.s.fNestedPaging
13940	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
13941	\|\| pVCpu->hm.s.fUsingDebugLoop);
13942	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
13943	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13944	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
13945	Log4Func(("CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
13946	break;
13947	}
13948
13949	case 4:
13950	{
13951	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
13952	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13953	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
13954	Log4Func(("CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
13955	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
13956	break;
13957	}
13958
13959	case 8:
13960	{
13961	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
13962	Assert(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
13963	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13964	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
13965	break;
13966	}
13967	default:
13968	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual)));
13969	break;
13970	}
13971	break;
13972	}
13973
13974	case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
13975	{
13976	Assert( !pVM->hm.s.fNestedPaging
13977	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
13978	\|\| pVCpu->hm.s.fUsingDebugLoop
13979	\|\| VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 3);
13980	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
13981	Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 8
13982	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
13983
13984	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_GENREG(uExitQual),
13985	VMX_EXIT_QUAL_CRX_REGISTER(uExitQual));
13986	AssertMsg( rcStrict == VINF_SUCCESS
13987	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13988	#ifdef VBOX_WITH_STATISTICS
13989	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
13990	{
13991	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
13992	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
13993	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
13994	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
13995	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
13996	}
13997	#endif
13998	Log4Func(("CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
13999	VBOXSTRICTRC_VAL(rcStrict)));
14000	if (VMX_EXIT_QUAL_CRX_GENREG(uExitQual) == X86_GREG_xSP)
14001	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
14002	else
14003	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14004	break;
14005	}
14006
14007	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
14008	{
14009	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
14010	AssertMsg( rcStrict == VINF_SUCCESS
14011	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14012
14013	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14014	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
14015	Log4Func(("CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
14016	break;
14017	}
14018
14019	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
14020	{
14021	/* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
14022	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
14023	AssertRCReturn(rc, rc);
14024	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual),
14025	pVmxTransient->uGuestLinearAddr);
14026	AssertMsg( rcStrict == VINF_SUCCESS
14027	\|\| rcStrict == VINF_IEM_RAISED_XCPT
14028	, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14029
14030	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14031	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
14032	Log4Func(("LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
14033	break;
14034	}
14035
14036	default:
14037	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
14038	VERR_VMX_UNEXPECTED_EXCEPTION);
14039	}
14040
14041	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
14042	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
14043	if (rcStrict == VINF_IEM_RAISED_XCPT)
14044	{
14045	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14046	rcStrict = VINF_SUCCESS;
14047	}
14048
14049	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
14050	NOREF(pVM);
14051	return rcStrict;
14052	}
14053
14054
14055	/**
14056	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
14057	* VM-exit.
14058	*/
14059	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14060	{
14061	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14062	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
14063
14064	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14065	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14066	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14067	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14068	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
14069	\| CPUMCTX_EXTRN_EFER);
14070	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
14071	AssertRCReturn(rc, rc);
14072
14073	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
14074	uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
14075	uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQual);
14076	bool fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
14077	bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
14078	bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
14079	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
14080	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
14081
14082	/*
14083	* Update exit history to see if this exit can be optimized.
14084	*/
14085	VBOXSTRICTRC rcStrict;
14086	PCEMEXITREC pExitRec = NULL;
14087	if ( !fGstStepping
14088	&& !fDbgStepping)
14089	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14090	!fIOString
14091	? !fIOWrite
14092	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
14093	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
14094	: !fIOWrite
14095	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
14096	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
14097	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14098	if (!pExitRec)
14099	{
14100	/* I/O operation lookup arrays. */
14101	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
14102	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
14103	uint32_t const cbValue = s_aIOSizes[uIOWidth];
14104	uint32_t const cbInstr = pVmxTransient->cbInstr;
14105	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
14106	PVM pVM = pVCpu->CTX_SUFF(pVM);
14107	if (fIOString)
14108	{
14109	/*
14110	* INS/OUTS - I/O String instruction.
14111	*
14112	* Use instruction-information if available, otherwise fall back on
14113	* interpreting the instruction.
14114	*/
14115	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14116	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
14117	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
14118	if (fInsOutsInfo)
14119	{
14120	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14121	AssertRCReturn(rc2, rc2);
14122	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
14123	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
14124	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
14125	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
14126	if (fIOWrite)
14127	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
14128	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
14129	else
14130	{
14131	/*
14132	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
14133	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
14134	* See Intel Instruction spec. for "INS".
14135	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
14136	*/
14137	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
14138	}
14139	}
14140	else
14141	rcStrict = IEMExecOne(pVCpu);
14142
14143	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14144	fUpdateRipAlready = true;
14145	}
14146	else
14147	{
14148	/*
14149	* IN/OUT - I/O instruction.
14150	*/
14151	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14152	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
14153	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
14154	if (fIOWrite)
14155	{
14156	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
14157	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
14158	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14159	&& !pCtx->eflags.Bits.u1TF)
14160	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
14161	}
14162	else
14163	{
14164	uint32_t u32Result = 0;
14165	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
14166	if (IOM_SUCCESS(rcStrict))
14167	{
14168	/* Save result of I/O IN instr. in AL/AX/EAX. */
14169	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
14170	}
14171	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14172	&& !pCtx->eflags.Bits.u1TF)
14173	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
14174	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
14175	}
14176	}
14177
14178	if (IOM_SUCCESS(rcStrict))
14179	{
14180	if (!fUpdateRipAlready)
14181	{
14182	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
14183	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14184	}
14185
14186	/*
14187	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
14188	* while booting Fedora 17 64-bit guest.
14189	*
14190	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
14191	*/
14192	if (fIOString)
14193	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
14194
14195	/*
14196	* If any I/O breakpoints are armed, we need to check if one triggered
14197	* and take appropriate action.
14198	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
14199	*/
14200	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
14201	AssertRCReturn(rc, rc);
14202
14203	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
14204	* execution engines about whether hyper BPs and such are pending. */
14205	uint32_t const uDr7 = pCtx->dr[7];
14206	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
14207	&& X86_DR7_ANY_RW_IO(uDr7)
14208	&& (pCtx->cr4 & X86_CR4_DE))
14209	\|\| DBGFBpIsHwIoArmed(pVM)))
14210	{
14211	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
14212
14213	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
14214	VMMRZCallRing3Disable(pVCpu);
14215	HM_DISABLE_PREEMPT(pVCpu);
14216
14217	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
14218
14219	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
14220	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
14221	{
14222	/* Raise #DB. */
14223	if (fIsGuestDbgActive)
14224	ASMSetDR6(pCtx->dr[6]);
14225	if (pCtx->dr[7] != uDr7)
14226	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
14227
14228	hmR0VmxSetPendingXcptDB(pVCpu);
14229	}
14230	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
14231	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
14232	else if ( rcStrict2 != VINF_SUCCESS
14233	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
14234	rcStrict = rcStrict2;
14235	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
14236
14237	HM_RESTORE_PREEMPT();
14238	VMMRZCallRing3Enable(pVCpu);
14239	}
14240	}
14241
14242	#ifdef VBOX_STRICT
14243	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14244	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
14245	Assert(!fIOWrite);
14246	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14247	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
14248	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
14249	Assert(fIOWrite);
14250	else
14251	{
14252	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
14253	* statuses, that the VMM device and some others may return. See
14254	* IOM_SUCCESS() for guidance. */
14255	AssertMsg( RT_FAILURE(rcStrict)
14256	\|\| rcStrict == VINF_SUCCESS
14257	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
14258	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
14259	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
14260	\|\| rcStrict == VINF_EM_RAW_TO_R3
14261	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14262	# endif
14263	}
14264	#endif
14265	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
14266	}
14267	else
14268	{
14269	/*
14270	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14271	*/
14272	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14273	AssertRCReturn(rc2, rc2);
14274	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
14275	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
14276	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
14277	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14278	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
14279	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
14280
14281	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14282	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14283
14284	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14285	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14286	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14287	}
14288	return rcStrict;
14289	}
14290
14291
14292	/**
14293	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
14294	* VM-exit.
14295	*/
14296	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14297	{
14298	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14299
14300	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
14301	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14302	AssertRCReturn(rc, rc);
14303	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
14304	{
14305	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14306	AssertRCReturn(rc, rc);
14307	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
14308	{
14309	uint32_t uErrCode;
14310	RTGCUINTPTR GCPtrFaultAddress;
14311	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
14312	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
14313	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
14314	if (fErrorCodeValid)
14315	{
14316	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14317	AssertRCReturn(rc, rc);
14318	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
14319	}
14320	else
14321	uErrCode = 0;
14322
14323	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
14324	&& uVector == X86_XCPT_PF)
14325	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
14326	else
14327	GCPtrFaultAddress = 0;
14328
14329	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14330	AssertRCReturn(rc, rc);
14331
14332	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
14333	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
14334
14335	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
14336	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14337	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14338	}
14339	}
14340
14341	/* Fall back to the interpreter to emulate the task-switch. */
14342	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14343	return VERR_EM_INTERPRETER;
14344	}
14345
14346
14347	/**
14348	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
14349	*/
14350	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14351	{
14352	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14353
14354	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14355	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
14356	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14357	AssertRCReturn(rc, rc);
14358	return VINF_EM_DBG_STEPPED;
14359	}
14360
14361
14362	/**
14363	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
14364	*/
14365	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14366	{
14367	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14368	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
14369
14370	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14371	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14372	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14373	{
14374	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
14375	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14376	{
14377	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14378	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14379	}
14380	}
14381	else
14382	{
14383	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14384	rcStrict1 = VINF_SUCCESS;
14385	return rcStrict1;
14386	}
14387
14388	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
14389	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14390	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14391	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14392	AssertRCReturn(rc, rc);
14393
14394	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
14395	uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
14396	VBOXSTRICTRC rcStrict2;
14397	switch (uAccessType)
14398	{
14399	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
14400	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
14401	{
14402	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
14403	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
14404	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
14405
14406	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
14407	GCPhys &= PAGE_BASE_GC_MASK;
14408	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
14409	PVM pVM = pVCpu->CTX_SUFF(pVM);
14410	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
14411	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
14412
14413	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14414	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
14415	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
14416	CPUMCTX2CORE(pCtx), GCPhys);
14417	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14418	if ( rcStrict2 == VINF_SUCCESS
14419	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14420	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14421	{
14422	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14423	\| HM_CHANGED_GUEST_APIC_TPR);
14424	rcStrict2 = VINF_SUCCESS;
14425	}
14426	break;
14427	}
14428
14429	default:
14430	Log4Func(("uAccessType=%#x\n", uAccessType));
14431	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
14432	break;
14433	}
14434
14435	if (rcStrict2 != VINF_SUCCESS)
14436	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
14437	return rcStrict2;
14438	}
14439
14440
14441	/**
14442	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
14443	* VM-exit.
14444	*/
14445	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14446	{
14447	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14448
14449	/* We should -not- get this VM-exit if the guest's debug registers were active. */
14450	if (pVmxTransient->fWasGuestDebugStateActive)
14451	{
14452	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
14453	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14454	}
14455
14456	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14457	if ( !pVCpu->hm.s.fSingleInstruction
14458	&& !pVmxTransient->fWasHyperDebugStateActive)
14459	{
14460	Assert(!DBGFIsStepping(pVCpu));
14461	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
14462
14463	/* Don't intercept MOV DRx any more. */
14464	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
14465	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14466	AssertRCReturn(rc, rc);
14467
14468	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
14469	VMMRZCallRing3Disable(pVCpu);
14470	HM_DISABLE_PREEMPT(pVCpu);
14471
14472	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
14473	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
14474	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
14475
14476	HM_RESTORE_PREEMPT();
14477	VMMRZCallRing3Enable(pVCpu);
14478
14479	#ifdef VBOX_WITH_STATISTICS
14480	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14481	AssertRCReturn(rc, rc);
14482	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14483	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14484	else
14485	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14486	#endif
14487	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
14488	return VINF_SUCCESS;
14489	}
14490
14491	/*
14492	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
14493	* The EFER MSR is always up-to-date.
14494	* Update the segment registers and DR7 from the CPU.
14495	*/
14496	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14497	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14498	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
14499	AssertRCReturn(rc, rc);
14500	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
14501
14502	PVM pVM = pVCpu->CTX_SUFF(pVM);
14503	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14504	{
14505	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14506	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
14507	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
14508	if (RT_SUCCESS(rc))
14509	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
14510	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14511	}
14512	else
14513	{
14514	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14515	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
14516	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
14517	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14518	}
14519
14520	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
14521	if (RT_SUCCESS(rc))
14522	{
14523	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14524	AssertRCReturn(rc2, rc2);
14525	return VINF_SUCCESS;
14526	}
14527	return rc;
14528	}
14529
14530
14531	/**
14532	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
14533	* Conditional VM-exit.
14534	*/
14535	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14536	{
14537	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14538	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14539
14540	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14541	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14542	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14543	{
14544	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
14545	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
14546	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14547	{
14548	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14549	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14550	}
14551	}
14552	else
14553	{
14554	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14555	rcStrict1 = VINF_SUCCESS;
14556	return rcStrict1;
14557	}
14558
14559	/*
14560	* Get sufficent state and update the exit history entry.
14561	*/
14562	RTGCPHYS GCPhys;
14563	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14564	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14565	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14566	AssertRCReturn(rc, rc);
14567
14568	VBOXSTRICTRC rcStrict;
14569	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14570	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
14571	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14572	if (!pExitRec)
14573	{
14574	/*
14575	* If we succeed, resume guest execution.
14576	* If we fail in interpreting the instruction because we couldn't get the guest physical address
14577	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
14578	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
14579	* weird case. See @bugref{6043}.
14580	*/
14581	PVM pVM = pVCpu->CTX_SUFF(pVM);
14582	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14583	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
14584	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
14585	if ( rcStrict == VINF_SUCCESS
14586	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
14587	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
14588	{
14589	/* Successfully handled MMIO operation. */
14590	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14591	\| HM_CHANGED_GUEST_APIC_TPR);
14592	rcStrict = VINF_SUCCESS;
14593	}
14594	}
14595	else
14596	{
14597	/*
14598	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14599	*/
14600	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14601	AssertRCReturn(rc2, rc2);
14602
14603	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
14604	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
14605
14606	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14607	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14608
14609	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14610	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14611	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14612	}
14613	return VBOXSTRICTRC_TODO(rcStrict);
14614	}
14615
14616
14617	/**
14618	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
14619	* VM-exit.
14620	*/
14621	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14622	{
14623	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14624	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14625
14626	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14627	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14628	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14629	{
14630	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
14631	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14632	Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
14633	}
14634	else
14635	{
14636	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14637	rcStrict1 = VINF_SUCCESS;
14638	return rcStrict1;
14639	}
14640
14641	RTGCPHYS GCPhys;
14642	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14643	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14644	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14645	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14646	AssertRCReturn(rc, rc);
14647
14648	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
14649	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
14650
14651	RTGCUINT uErrorCode = 0;
14652	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
14653	uErrorCode \|= X86_TRAP_PF_ID;
14654	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
14655	uErrorCode \|= X86_TRAP_PF_RW;
14656	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
14657	uErrorCode \|= X86_TRAP_PF_P;
14658
14659	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
14660
14661
14662	/* Handle the pagefault trap for the nested shadow table. */
14663	PVM pVM = pVCpu->CTX_SUFF(pVM);
14664	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14665
14666	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
14667	pCtx->cs.Sel, pCtx->rip));
14668
14669	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
14670	TRPMResetTrap(pVCpu);
14671
14672	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
14673	if ( rcStrict2 == VINF_SUCCESS
14674	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14675	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14676	{
14677	/* Successfully synced our nested page tables. */
14678	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
14679	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
14680	return VINF_SUCCESS;
14681	}
14682
14683	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14684	return rcStrict2;
14685	}
14686
14687	/** @} */
14688
14689	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14690	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14691	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14692
14693	/**
14694	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
14695	*/
14696	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14697	{
14698	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14699	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
14700
14701	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
14702	AssertRCReturn(rc, rc);
14703
14704	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
14705	{
14706	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
14707	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
14708
14709	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
14710	* provides VM-exit instruction length. If this causes problem later,
14711	* disassemble the instruction like it's done on AMD-V. */
14712	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14713	AssertRCReturn(rc2, rc2);
14714	return rc;
14715	}
14716
14717	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14718	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14719	return rc;
14720	}
14721
14722
14723	/**
14724	* VM-exit exception handler for \#BP (Breakpoint exception).
14725	*/
14726	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14727	{
14728	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14729	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
14730
14731	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14732	AssertRCReturn(rc, rc);
14733
14734	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14735	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14736	if (rc == VINF_EM_RAW_GUEST_TRAP)
14737	{
14738	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14739	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14740	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14741	AssertRCReturn(rc, rc);
14742
14743	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14744	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14745	}
14746
14747	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
14748	return rc;
14749	}
14750
14751
14752	/**
14753	* VM-exit exception handler for \#AC (alignment check exception).
14754	*/
14755	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14756	{
14757	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14758
14759	/*
14760	* Re-inject it. We'll detect any nesting before getting here.
14761	*/
14762	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14763	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14764	AssertRCReturn(rc, rc);
14765	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
14766
14767	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14768	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14769	return VINF_SUCCESS;
14770	}
14771
14772
14773	/**
14774	* VM-exit exception handler for \#DB (Debug exception).
14775	*/
14776	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14777	{
14778	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14779	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
14780
14781	/*
14782	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
14783	* for processing.
14784	*/
14785	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14786
14787	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
14788	uint64_t uDR6 = X86_DR6_INIT_VAL;
14789	uDR6 \|= (pVmxTransient->uExitQual & (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
14790
14791	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14792	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
14793	Log6Func(("rc=%Rrc\n", rc));
14794	if (rc == VINF_EM_RAW_GUEST_TRAP)
14795	{
14796	/*
14797	* The exception was for the guest. Update DR6, DR7.GD and
14798	* IA32_DEBUGCTL.LBR before forwarding it.
14799	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
14800	*/
14801	VMMRZCallRing3Disable(pVCpu);
14802	HM_DISABLE_PREEMPT(pVCpu);
14803
14804	pCtx->dr[6] &= ~X86_DR6_B_MASK;
14805	pCtx->dr[6] \|= uDR6;
14806	if (CPUMIsGuestDebugStateActive(pVCpu))
14807	ASMSetDR6(pCtx->dr[6]);
14808
14809	HM_RESTORE_PREEMPT();
14810	VMMRZCallRing3Enable(pVCpu);
14811
14812	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
14813	AssertRCReturn(rc, rc);
14814
14815	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
14816	pCtx->dr[7] &= ~X86_DR7_GD;
14817
14818	/* Paranoia. */
14819	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
14820	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
14821
14822	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
14823	AssertRCReturn(rc, rc);
14824
14825	/*
14826	* Raise #DB in the guest.
14827	*
14828	* It is important to reflect exactly what the VM-exit gave us (preserving the
14829	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
14830	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
14831	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
14832	*
14833	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
14834	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
14835	*/
14836	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14837	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14838	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14839	AssertRCReturn(rc, rc);
14840	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14841	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14842	return VINF_SUCCESS;
14843	}
14844
14845	/*
14846	* Not a guest trap, must be a hypervisor related debug event then.
14847	* Update DR6 in case someone is interested in it.
14848	*/
14849	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
14850	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
14851	CPUMSetHyperDR6(pVCpu, uDR6);
14852
14853	return rc;
14854	}
14855
14856
14857	/**
14858	* Hacks its way around the lovely mesa driver's backdoor accesses.
14859	*
14860	* @sa hmR0SvmHandleMesaDrvGp
14861	*/
14862	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14863	{
14864	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
14865	RT_NOREF(pCtx);
14866
14867	/* For now we'll just skip the instruction. */
14868	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14869	}
14870
14871
14872	/**
14873	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
14874	* backdoor logging w/o checking what it is running inside.
14875	*
14876	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
14877	* backdoor port and magic numbers loaded in registers.
14878	*
14879	* @returns true if it is, false if it isn't.
14880	* @sa hmR0SvmIsMesaDrvGp
14881	*/
14882	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14883	{
14884	/* 0xed: IN eAX,dx */
14885	uint8_t abInstr[1];
14886	if (pVmxTransient->cbInstr != sizeof(abInstr))
14887	return false;
14888
14889	/* Check that it is #GP(0). */
14890	if (pVmxTransient->uExitIntErrorCode != 0)
14891	return false;
14892
14893	/* Check magic and port. */
14894	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
14895	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
14896	if (pCtx->rax != UINT32_C(0x564d5868))
14897	return false;
14898	if (pCtx->dx != UINT32_C(0x5658))
14899	return false;
14900
14901	/* Flat ring-3 CS. */
14902	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
14903	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
14904	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
14905	if (pCtx->cs.Attr.n.u2Dpl != 3)
14906	return false;
14907	if (pCtx->cs.u64Base != 0)
14908	return false;
14909
14910	/* Check opcode. */
14911	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
14912	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
14913	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
14914	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
14915	if (RT_FAILURE(rc))
14916	return false;
14917	if (abInstr[0] != 0xed)
14918	return false;
14919
14920	return true;
14921	}
14922
14923
14924	/**
14925	* VM-exit exception handler for \#GP (General-protection exception).
14926	*
14927	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
14928	*/
14929	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14930	{
14931	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14932	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
14933
14934	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14935	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14936	if (pVmcsInfo->RealMode.fRealOnV86Active)
14937	{ /* likely */ }
14938	else
14939	{
14940	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14941	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv);
14942	#endif
14943	/* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
14944	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14945	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14946	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14947	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14948	AssertRCReturn(rc, rc);
14949	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
14950	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
14951
14952	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
14953	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
14954	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14955	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14956	else
14957	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
14958	return rc;
14959	}
14960
14961	Assert(CPUMIsGuestInRealModeEx(pCtx));
14962	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
14963
14964	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14965	AssertRCReturn(rc, rc);
14966
14967	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
14968	if (rcStrict == VINF_SUCCESS)
14969	{
14970	if (!CPUMIsGuestInRealModeEx(pCtx))
14971	{
14972	/*
14973	* The guest is no longer in real-mode, check if we can continue executing the
14974	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
14975	*/
14976	pVmcsInfo->RealMode.fRealOnV86Active = false;
14977	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
14978	{
14979	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
14980	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14981	}
14982	else
14983	{
14984	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
14985	rcStrict = VINF_EM_RESCHEDULE;
14986	}
14987	}
14988	else
14989	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14990	}
14991	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14992	{
14993	rcStrict = VINF_SUCCESS;
14994	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14995	}
14996	return VBOXSTRICTRC_VAL(rcStrict);
14997	}
14998
14999
15000	/**
15001	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
15002	* the exception reported in the VMX transient structure back into the VM.
15003	*
15004	* @remarks Requires uExitIntInfo in the VMX transient structure to be
15005	* up-to-date.
15006	*/
15007	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15008	{
15009	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15010	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
15011	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15012	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active,
15013	("uVector=%#x u32XcptBitmap=%#X32\n",
15014	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
15015	NOREF(pVmcsInfo);
15016	#endif
15017
15018	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
15019	hmR0VmxCheckExitDueToEventDelivery(). */
15020	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15021	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15022	AssertRCReturn(rc, rc);
15023	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15024
15025	#ifdef DEBUG_ramshankar
15026	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
15027	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n",
15028	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pCtx->cs.Sel, pCtx->rip));
15029	#endif
15030
15031	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15032	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15033	return VINF_SUCCESS;
15034	}
15035
15036
15037	/**
15038	* VM-exit exception handler for \#PF (Page-fault exception).
15039	*/
15040	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15041	{
15042	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15043	PVM pVM = pVCpu->CTX_SUFF(pVM);
15044	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15045	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15046	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15047	AssertRCReturn(rc, rc);
15048
15049	if (!pVM->hm.s.fNestedPaging)
15050	{ /* likely */ }
15051	else
15052	{
15053	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
15054	Assert(pVCpu->hm.s.fUsingDebugLoop);
15055	#endif
15056	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
15057	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
15058	{
15059	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15060	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
15061	}
15062	else
15063	{
15064	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15065	hmR0VmxSetPendingXcptDF(pVCpu);
15066	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
15067	}
15068	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15069	return rc;
15070	}
15071
15072	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
15073	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
15074	if (pVmxTransient->fVectoringPF)
15075	{
15076	Assert(pVCpu->hm.s.Event.fPending);
15077	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15078	}
15079
15080	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15081	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15082	AssertRCReturn(rc, rc);
15083
15084	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
15085	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
15086
15087	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
15088	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
15089
15090	Log4Func(("#PF: rc=%Rrc\n", rc));
15091	if (rc == VINF_SUCCESS)
15092	{
15093	/*
15094	* This is typically a shadow page table sync or a MMIO instruction. But we may have
15095	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
15096	*/
15097	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15098	TRPMResetTrap(pVCpu);
15099	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
15100	return rc;
15101	}
15102
15103	if (rc == VINF_EM_RAW_GUEST_TRAP)
15104	{
15105	if (!pVmxTransient->fVectoringDoublePF)
15106	{
15107	/* It's a guest page fault and needs to be reflected to the guest. */
15108	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
15109	TRPMResetTrap(pVCpu);
15110	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
15111	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15112	uGstErrorCode, pVmxTransient->uExitQual);
15113	}
15114	else
15115	{
15116	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15117	TRPMResetTrap(pVCpu);
15118	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
15119	hmR0VmxSetPendingXcptDF(pVCpu);
15120	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
15121	}
15122
15123	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15124	return VINF_SUCCESS;
15125	}
15126
15127	TRPMResetTrap(pVCpu);
15128	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
15129	return rc;
15130	}
15131
15132	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15133	/** @name VMX instruction handlers.
15134	* @{
15135	*/
15136	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15137	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VMX instructions VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15138	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15139
15140	/**
15141	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
15142	*/
15143	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15144	{
15145	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15146
15147	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15148	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15149	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15150	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15151	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15152	AssertRCReturn(rc, rc);
15153
15154	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15155
15156	VMXVEXITINFO ExitInfo;
15157	RT_ZERO(ExitInfo);
15158	ExitInfo.uReason = pVmxTransient->uExitReason;
15159	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15160	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15161	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15162	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15163
15164	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
15165	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15166	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15167	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15168	{
15169	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15170	rcStrict = VINF_SUCCESS;
15171	}
15172	return rcStrict;
15173	}
15174
15175
15176	/**
15177	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
15178	*/
15179	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15180	{
15181	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15182
15183	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
15184	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15185	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15186	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15187	AssertRCReturn(rc, rc);
15188
15189	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15190
15191	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
15192	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15193	{
15194	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15195	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15196	}
15197	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15198	return rcStrict;
15199	}
15200
15201
15202	/**
15203	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
15204	*/
15205	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15206	{
15207	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15208
15209	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15210	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15211	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15212	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15213	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15214	AssertRCReturn(rc, rc);
15215
15216	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15217
15218	VMXVEXITINFO ExitInfo;
15219	RT_ZERO(ExitInfo);
15220	ExitInfo.uReason = pVmxTransient->uExitReason;
15221	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15222	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15223	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15224	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15225
15226	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
15227	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15228	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15229	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15230	{
15231	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15232	rcStrict = VINF_SUCCESS;
15233	}
15234	return rcStrict;
15235	}
15236
15237
15238	/**
15239	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
15240	*/
15241	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15242	{
15243	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15244
15245	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15246	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15247	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15248	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15249	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15250	AssertRCReturn(rc, rc);
15251
15252	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15253
15254	VMXVEXITINFO ExitInfo;
15255	RT_ZERO(ExitInfo);
15256	ExitInfo.uReason = pVmxTransient->uExitReason;
15257	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15258	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15259	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15260	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15261
15262	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
15263	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15264	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15265	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15266	{
15267	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15268	rcStrict = VINF_SUCCESS;
15269	}
15270	return rcStrict;
15271	}
15272
15273
15274	/**
15275	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Unconditional VM-exit.
15276	*/
15277	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15278	{
15279	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15280
15281	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15282	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15283	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15284	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15285	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15286	AssertRCReturn(rc, rc);
15287
15288	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15289
15290	VMXVEXITINFO ExitInfo;
15291	RT_ZERO(ExitInfo);
15292	ExitInfo.uReason = pVmxTransient->uExitReason;
15293	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15294	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15295	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15296	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15297	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15298
15299	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
15300	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15301	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15302	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15303	{
15304	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15305	rcStrict = VINF_SUCCESS;
15306	}
15307	return rcStrict;
15308	}
15309
15310
15311	/**
15312	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
15313	*/
15314	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15315	{
15316	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15317
15318	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
15319	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15320	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15321	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15322	AssertRCReturn(rc, rc);
15323
15324	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15325
15326	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
15327	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15328	{
15329	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15330	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15331	}
15332	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15333	return rcStrict;
15334	}
15335
15336
15337	/**
15338	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Unconditional VM-exit.
15339	*/
15340	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15341	{
15342	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15343
15344	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15345	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15346	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15347	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15348	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15349	AssertRCReturn(rc, rc);
15350
15351	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15352
15353	VMXVEXITINFO ExitInfo;
15354	RT_ZERO(ExitInfo);
15355	ExitInfo.uReason = pVmxTransient->uExitReason;
15356	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15357	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15358	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15359	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15360	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15361
15362	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
15363	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15364	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15365	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15366	{
15367	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15368	rcStrict = VINF_SUCCESS;
15369	}
15370	return rcStrict;
15371	}
15372
15373
15374	/**
15375	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
15376	*/
15377	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15378	{
15379	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15380
15381	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15382	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
15383	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15384	AssertRCReturn(rc, rc);
15385
15386	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15387
15388	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
15389	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15390	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
15391	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15392	{
15393	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15394	rcStrict = VINF_SUCCESS;
15395	}
15396	return rcStrict;
15397	}
15398
15399
15400	/**
15401	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
15402	*/
15403	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15404	{
15405	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15406
15407	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15408	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15409	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15410	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15411	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15412	AssertRCReturn(rc, rc);
15413
15414	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15415
15416	VMXVEXITINFO ExitInfo;
15417	RT_ZERO(ExitInfo);
15418	ExitInfo.uReason = pVmxTransient->uExitReason;
15419	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15420	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15421	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15422	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15423
15424	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
15425	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15426	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15427	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15428	{
15429	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15430	rcStrict = VINF_SUCCESS;
15431	}
15432	return rcStrict;
15433	}
15434
15435	/** @} */
15436	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
15437

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 78230

Download in other formats: