HMVMXR0.cpp@ 78221

Last change on this file since 78221 was 78221, checked in by vboxsync, 6 years ago
VMM/HMVMXR0: Nested VMX: bugref:9180 Unused var fix.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 641.6 KB

Line
1	/* $Id: HMVMXR0.cpp 78221 2019-04-20 04:17:42Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27
28	#include <VBox/vmm/pdmapi.h>
29	#include <VBox/vmm/dbgf.h>
30	#include <VBox/vmm/iem.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#include <VBox/vmm/em.h>
35	#include <VBox/vmm/gim.h>
36	#include <VBox/vmm/apic.h>
37	#ifdef VBOX_WITH_REM
38	# include <VBox/vmm/rem.h>
39	#endif
40	#include "HMInternal.h"
41	#include <VBox/vmm/vm.h>
42	#include <VBox/vmm/hmvmxinline.h>
43	#include "HMVMXR0.h"
44	#include "dtrace/VBoxVMM.h"
45
46	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
47	#ifdef DEBUG_ramshankar
48	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
49	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
50	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
51	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
52	# define HMVMX_ALWAYS_TRAP_PF
53	# define HMVMX_ALWAYS_FLUSH_TLB
54	# define HMVMX_ALWAYS_SWAP_EFER
55	#endif
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	/** Use the function table. */
62	#define HMVMX_USE_FUNCTION_TABLE
63
64	/** Determine which tagged-TLB flush handler to use. */
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70	/** @name HMVMX_READ_XXX
71	* Flags to skip redundant reads of some common VMCS fields that are not part of
72	* the guest-CPU or VCPU state but are needed while handling VM-exits.
73	*/
74	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
75	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
76	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
77	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
78	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
79	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
80	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
81	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
82	/** @} */
83
84	/**
85	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
86	* guest using hardware-assisted VMX.
87	*
88	* This excludes state like GPRs (other than RSP) which are always are
89	* swapped and restored across the world-switch and also registers like EFER,
90	* MSR which cannot be modified by the guest without causing a VM-exit.
91	*/
92	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
93	\| CPUMCTX_EXTRN_RFLAGS \
94	\| CPUMCTX_EXTRN_RSP \
95	\| CPUMCTX_EXTRN_SREG_MASK \
96	\| CPUMCTX_EXTRN_TABLE_MASK \
97	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
98	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
99	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
100	\| CPUMCTX_EXTRN_TSC_AUX \
101	\| CPUMCTX_EXTRN_OTHER_MSRS \
102	\| CPUMCTX_EXTRN_CR0 \
103	\| CPUMCTX_EXTRN_CR3 \
104	\| CPUMCTX_EXTRN_CR4 \
105	\| CPUMCTX_EXTRN_DR7 \
106	\| CPUMCTX_EXTRN_HM_VMX_MASK)
107
108	/**
109	* Exception bitmap mask for real-mode guests (real-on-v86).
110	*
111	* We need to intercept all exceptions manually except:
112	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
113	* due to bugs in Intel CPUs.
114	* - \#PF need not be intercepted even in real-mode if we have nested paging
115	* support.
116	*/
117	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
118	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
119	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
120	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
121	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
122	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
123	\| RT_BIT(X86_XCPT_XF))
124
125	/** Maximum VM-instruction error number. */
126	#define HMVMX_INSTR_ERROR_MAX 28
127
128	/** Profiling macro. */
129	#ifdef HM_PROFILE_EXIT_DISPATCH
130	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
131	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
132	#else
133	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
134	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
135	#endif
136
137	/** Assert that preemption is disabled or covered by thread-context hooks. */
138	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
139	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
140
141	/** Assert that we haven't migrated CPUs when thread-context hooks are not
142	* used. */
143	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
144	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
145	("Illegal migration! Entered on CPU %u Current %u\n", \
146	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
147
148	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
149	* context. */
150	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
151	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
152	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
153
154	/** Helper macro for VM-exit handlers called unexpectedly. */
155	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
156	do { \
157	(a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
158	return VERR_VMX_UNEXPECTED_EXIT; \
159	} while (0)
160
161	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
162	/** Macro that does the necessary privilege checks and intercepted VM-exits for
163	* guests that attempted to execute a VMX instruction. */
164	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
165	do \
166	{ \
167	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
168	if (rcStrictTmp == VINF_SUCCESS) \
169	{ /* likely */ } \
170	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
171	{ \
172	Assert((a_pVCpu)->hm.s.Event.fPending); \
173	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
174	return VINF_SUCCESS; \
175	} \
176	else \
177	{ \
178	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
179	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
180	} \
181	} while (0)
182
183	/** Macro that decodes a memory operand for an instruction VM-exit. */
184	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
185	do \
186	{ \
187	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
188	(a_pGCPtrEffAddr)); \
189	if (rcStrictTmp == VINF_SUCCESS) \
190	{ /* likely */ } \
191	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
192	{ \
193	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
194	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
195	NOREF(uXcptTmp); \
196	return VINF_SUCCESS; \
197	} \
198	else \
199	{ \
200	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
201	return rcStrictTmp; \
202	} \
203	} while (0)
204
205	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
206
207
208	/*********************************************************************************************************************************
209	* Structures and Typedefs *
210	*********************************************************************************************************************************/
211	/**
212	* VMX transient state.
213	*
214	* A state structure for holding miscellaneous information across
215	* VMX non-root operation and restored after the transition.
216	*/
217	typedef struct VMXTRANSIENT
218	{
219	/** The host's rflags/eflags. */
220	RTCCUINTREG fEFlags;
221	#if HC_ARCH_BITS == 32
222	uint32_t u32Alignment0;
223	#endif
224	/** The guest's TPR value used for TPR shadowing. */
225	uint8_t u8GuestTpr;
226	/** Alignment. */
227	uint8_t abAlignment0[7];
228
229	/** The basic VM-exit reason. */
230	uint16_t uExitReason;
231	/** Alignment. */
232	uint16_t u16Alignment0;
233	/** The VM-exit interruption error code. */
234	uint32_t uExitIntErrorCode;
235	/** The VM-exit exit code qualification. */
236	uint64_t uExitQual;
237	/** The Guest-linear address. */
238	uint64_t uGuestLinearAddr;
239
240	/** The VM-exit interruption-information field. */
241	uint32_t uExitIntInfo;
242	/** The VM-exit instruction-length field. */
243	uint32_t cbInstr;
244	/** The VM-exit instruction-information field. */
245	VMXEXITINSTRINFO ExitInstrInfo;
246	/** Whether the VM-entry failed or not. */
247	bool fVMEntryFailed;
248	/** Whether we are currently executing a nested-guest. */
249	bool fIsNestedGuest;
250	/** Alignment. */
251	uint8_t abAlignment1[2];
252
253	/** The VM-entry interruption-information field. */
254	uint32_t uEntryIntInfo;
255	/** The VM-entry exception error code field. */
256	uint32_t uEntryXcptErrorCode;
257	/** The VM-entry instruction length field. */
258	uint32_t cbEntryInstr;
259
260	/** IDT-vectoring information field. */
261	uint32_t uIdtVectoringInfo;
262	/** IDT-vectoring error code. */
263	uint32_t uIdtVectoringErrorCode;
264
265	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
266	uint32_t fVmcsFieldsRead;
267
268	/** Whether the guest debug state was active at the time of VM-exit. */
269	bool fWasGuestDebugStateActive;
270	/** Whether the hyper debug state was active at the time of VM-exit. */
271	bool fWasHyperDebugStateActive;
272	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
273	bool fUpdatedTscOffsettingAndPreemptTimer;
274	/** Whether the VM-exit was caused by a page-fault during delivery of a
275	* contributory exception or a page-fault. */
276	bool fVectoringDoublePF;
277	/** Whether the VM-exit was caused by a page-fault during delivery of an
278	* external interrupt or NMI. */
279	bool fVectoringPF;
280	bool afAlignment0[3];
281
282	/** The VMCS info. object. */
283	PVMXVMCSINFO pVmcsInfo;
284	} VMXTRANSIENT;
285	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
286	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
287	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
288	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
289	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
290	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
291	/** Pointer to VMX transient state. */
292	typedef VMXTRANSIENT *PVMXTRANSIENT;
293
294	/**
295	* Memory operand read or write access.
296	*/
297	typedef enum VMXMEMACCESS
298	{
299	VMXMEMACCESS_READ = 0,
300	VMXMEMACCESS_WRITE = 1
301	} VMXMEMACCESS;
302
303	/**
304	* VMX VM-exit handler.
305	*
306	* @returns Strict VBox status code (i.e. informational status codes too).
307	* @param pVCpu The cross context virtual CPU structure.
308	* @param pVmxTransient The VMX-transient structure.
309	*/
310	#ifndef HMVMX_USE_FUNCTION_TABLE
311	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
312	#else
313	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
314	/** Pointer to VM-exit handler. */
315	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
316	#endif
317
318	/**
319	* VMX VM-exit handler, non-strict status code.
320	*
321	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
322	*
323	* @returns VBox status code, no informational status code returned.
324	* @param pVCpu The cross context virtual CPU structure.
325	* @param pVmxTransient The VMX-transient structure.
326	*
327	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
328	* use of that status code will be replaced with VINF_EM_SOMETHING
329	* later when switching over to IEM.
330	*/
331	#ifndef HMVMX_USE_FUNCTION_TABLE
332	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
333	#else
334	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
335	#endif
336
337
338	/*********************************************************************************************************************************
339	* Internal Functions *
340	*********************************************************************************************************************************/
341	#ifndef HMVMX_USE_FUNCTION_TABLE
342	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
343	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
344	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
345	#else
346	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
347	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
348	#endif
349	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
350	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
351	#endif
352
353	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
354	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
355	static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
356	#endif
357
358	/** @name VM-exit handlers.
359	* @{
360	*/
361	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
362	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
363	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
364	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
365	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
366	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
367	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
368	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
369	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
370	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
371	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
372	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
373	static FNVMXEXITHANDLER hmR0VmxExitHlt;
374	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
375	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
376	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
377	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
378	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
379	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
380	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
381	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
382	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
383	static FNVMXEXITHANDLER hmR0VmxExitVmread;
384	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
385	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
386	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
387	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
388	#endif
389	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
390	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
391	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
392	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
393	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
394	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
395	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
396	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
397	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
398	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
399	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
400	static FNVMXEXITHANDLER hmR0VmxExitMwait;
401	static FNVMXEXITHANDLER hmR0VmxExitMtf;
402	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
403	static FNVMXEXITHANDLER hmR0VmxExitPause;
404	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
405	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
406	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
407	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
408	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
409	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
410	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
411	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
412	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
413	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
414	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
415	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
416	/** @} */
417
418	/** @name Helpers for hardware exceptions VM-exit handlers.
419	* @{
420	*/
421	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
422	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
423	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
424	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
425	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
426	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
427	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
428	/** @} */
429
430
431	/*********************************************************************************************************************************
432	* Global Variables *
433	*********************************************************************************************************************************/
434	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
435	static const uint32_t g_aVmcsCacheSegBase[] =
436	{
437	VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
438	VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
439	VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
440	VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
441	VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
442	VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
443	};
444	AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
445	#endif
446	static const uint32_t g_aVmcsSegBase[] =
447	{
448	VMX_VMCS_GUEST_ES_BASE,
449	VMX_VMCS_GUEST_CS_BASE,
450	VMX_VMCS_GUEST_SS_BASE,
451	VMX_VMCS_GUEST_DS_BASE,
452	VMX_VMCS_GUEST_FS_BASE,
453	VMX_VMCS_GUEST_GS_BASE
454	};
455	static const uint32_t g_aVmcsSegSel[] =
456	{
457	VMX_VMCS16_GUEST_ES_SEL,
458	VMX_VMCS16_GUEST_CS_SEL,
459	VMX_VMCS16_GUEST_SS_SEL,
460	VMX_VMCS16_GUEST_DS_SEL,
461	VMX_VMCS16_GUEST_FS_SEL,
462	VMX_VMCS16_GUEST_GS_SEL
463	};
464	static const uint32_t g_aVmcsSegLimit[] =
465	{
466	VMX_VMCS32_GUEST_ES_LIMIT,
467	VMX_VMCS32_GUEST_CS_LIMIT,
468	VMX_VMCS32_GUEST_SS_LIMIT,
469	VMX_VMCS32_GUEST_DS_LIMIT,
470	VMX_VMCS32_GUEST_FS_LIMIT,
471	VMX_VMCS32_GUEST_GS_LIMIT
472	};
473	static const uint32_t g_aVmcsSegAttr[] =
474	{
475	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
476	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
477	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
478	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
479	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
480	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
481	};
482	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
483	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
484	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
485	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
486
487	#ifdef HMVMX_USE_FUNCTION_TABLE
488	/**
489	* VMX_EXIT dispatch table.
490	*/
491	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
492	{
493	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
494	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
495	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
496	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
497	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
498	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
499	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
500	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
501	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
502	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
503	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
504	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
505	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
506	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
507	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
508	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
509	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
510	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
511	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
512	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
513	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
514	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
515	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
516	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
517	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
518	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
519	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
520	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
521	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
522	#else
523	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
524	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
525	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
526	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
527	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
528	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
529	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
530	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
531	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
532	#endif
533	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
534	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
535	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
536	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
537	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
538	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
539	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
540	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
541	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
542	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
543	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
544	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
545	/* 40 UNDEFINED */ hmR0VmxExitPause,
546	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
547	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
548	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
549	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
550	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
551	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitXdtrAccess,
552	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitXdtrAccess,
553	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
554	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
555	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
556	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
557	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
558	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
559	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
560	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
561	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
562	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
563	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
564	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
565	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
566	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
567	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
568	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
569	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
570	};
571	#endif /* HMVMX_USE_FUNCTION_TABLE */
572
573	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
574	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
575	{
576	/* 0 */ "(Not Used)",
577	/* 1 */ "VMCALL executed in VMX root operation.",
578	/* 2 */ "VMCLEAR with invalid physical address.",
579	/* 3 */ "VMCLEAR with VMXON pointer.",
580	/* 4 */ "VMLAUNCH with non-clear VMCS.",
581	/* 5 */ "VMRESUME with non-launched VMCS.",
582	/* 6 */ "VMRESUME after VMXOFF",
583	/* 7 */ "VM-entry with invalid control fields.",
584	/* 8 */ "VM-entry with invalid host state fields.",
585	/* 9 */ "VMPTRLD with invalid physical address.",
586	/* 10 */ "VMPTRLD with VMXON pointer.",
587	/* 11 */ "VMPTRLD with incorrect revision identifier.",
588	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
589	/* 13 */ "VMWRITE to read-only VMCS component.",
590	/* 14 */ "(Not Used)",
591	/* 15 */ "VMXON executed in VMX root operation.",
592	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
593	/* 17 */ "VM-entry with non-launched executing VMCS.",
594	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
595	/* 19 */ "VMCALL with non-clear VMCS.",
596	/* 20 */ "VMCALL with invalid VM-exit control fields.",
597	/* 21 */ "(Not Used)",
598	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
599	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
600	/* 24 */ "VMCALL with invalid SMM-monitor features.",
601	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
602	/* 26 */ "VM-entry with events blocked by MOV SS.",
603	/* 27 */ "(Not Used)",
604	/* 28 */ "Invalid operand to INVEPT/INVVPID."
605	};
606	#endif /* VBOX_STRICT */
607
608
609	/**
610	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
611	*
612	* Any bit set in this mask is owned by the host/hypervisor and would cause a
613	* VM-exit when modified by the guest.
614	*
615	* @returns The static CR0 guest/host mask.
616	* @param pVCpu The cross context virtual CPU structure.
617	*/
618	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PVMCPU pVCpu)
619	{
620	/*
621	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
622	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
623	*/
624	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
625	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
626	* and @bugref{6944}. */
627	PVM pVM = pVCpu->CTX_SUFF(pVM);
628	return ( X86_CR0_PE
629	\| X86_CR0_NE
630	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
631	\| X86_CR0_PG
632	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
633	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
634	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
635	}
636
637
638	/**
639	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
640	*
641	* Any bit set in this mask is owned by the host/hypervisor and would cause a
642	* VM-exit when modified by the guest.
643	*
644	* @returns The static CR4 guest/host mask.
645	* @param pVCpu The cross context virtual CPU structure.
646	*/
647	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PVMCPU pVCpu)
648	{
649	/*
650	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
651	* these bits are reserved on hardware that does not support them. Since the
652	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
653	* these bits and handle it depending on whether we expose them to the guest.
654	*/
655	PVM pVM = pVCpu->CTX_SUFF(pVM);
656	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
657	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
658	return ( X86_CR4_VMXE
659	\| X86_CR4_VME
660	\| X86_CR4_PAE
661	\| X86_CR4_PGE
662	\| X86_CR4_PSE
663	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
664	\| (fPcid ? X86_CR4_PCIDE : 0));
665	}
666
667
668	/**
669	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
670	* area.
671	*
672	* @returns @c true if it's different, @c false otherwise.
673	* @param pVmcsInfo The VMCS info. object.
674	*/
675	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
676	{
677	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
678	&& pVmcsInfo->pvGuestMsrStore);
679	}
680
681
682	/**
683	* Adds one or more exceptions to the exception bitmap and commits it to the current
684	* VMCS.
685	*
686	* @returns VBox status code.
687	* @param pVmxTransient The VMX-transient structure.
688	* @param uXcptMask The exception(s) to add.
689	*/
690	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
691	{
692	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
693	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
694	if ((uXcptBitmap & uXcptMask) != uXcptMask)
695	{
696	uXcptBitmap \|= uXcptMask;
697	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
698	AssertRCReturn(rc, rc);
699	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
700	}
701	return VINF_SUCCESS;
702	}
703
704
705	/**
706	* Adds an exception to the exception bitmap and commits it to the current VMCS.
707	*
708	* @returns VBox status code.
709	* @param pVmxTransient The VMX-transient structure.
710	* @param uXcpt The exception to add.
711	*/
712	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
713	{
714	Assert(uXcpt <= X86_XCPT_LAST);
715	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
716	}
717
718
719	/**
720	* Remove one or more exceptions from the exception bitmap and commits it to the
721	* current VMCS.
722	*
723	* This takes care of not removing the exception intercept if a nested-guest
724	* requires the exception to be intercepted.
725	*
726	* @returns VBox status code.
727	* @param pVCpu The cross context virtual CPU structure.
728	* @param pVmxTransient The VMX-transient structure.
729	* @param uXcptMask The exception(s) to remove.
730	*/
731	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
732	{
733	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
734	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
735	if (u32XcptBitmap & uXcptMask)
736	{
737	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
738	if (!pVmxTransient->fIsNestedGuest)
739	{ /* likely */ }
740	else
741	{
742	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
743	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
744	}
745	#endif
746	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
747	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
748	\| RT_BIT(X86_XCPT_DE)
749	\| RT_BIT(X86_XCPT_NM)
750	\| RT_BIT(X86_XCPT_TS)
751	\| RT_BIT(X86_XCPT_UD)
752	\| RT_BIT(X86_XCPT_NP)
753	\| RT_BIT(X86_XCPT_SS)
754	\| RT_BIT(X86_XCPT_GP)
755	\| RT_BIT(X86_XCPT_PF)
756	\| RT_BIT(X86_XCPT_MF));
757	#elif defined(HMVMX_ALWAYS_TRAP_PF)
758	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
759	#endif
760	if (uXcptMask)
761	{
762	/* Validate we are not removing any essential exception intercepts. */
763	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
764	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
765	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
766
767	/* Remove it from the exception bitmap. */
768	u32XcptBitmap &= ~uXcptMask;
769
770	/* Commit and update the cache if necessary. */
771	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
772	{
773	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
774	AssertRCReturn(rc, rc);
775	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
776	}
777	}
778	}
779	return VINF_SUCCESS;
780	}
781
782
783	/**
784	* Remove an exceptions from the exception bitmap and commits it to the current
785	* VMCS.
786	*
787	* @returns VBox status code.
788	* @param pVCpu The cross context virtual CPU structure.
789	* @param pVmxTransient The VMX-transient structure.
790	* @param uXcpt The exception to remove.
791	*/
792	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
793	{
794	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
795	}
796
797
798	/**
799	* Loads the VMCS specified by the VMCS info. object.
800	*
801	* @returns VBox status code.
802	* @param pVmcsInfo The VMCS info. object.
803	*/
804	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
805	{
806	Assert(pVmcsInfo);
807	Assert(pVmcsInfo->HCPhysVmcs);
808	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
809
810	if (pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_CLEAR)
811	{
812	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
813	if (RT_SUCCESS(rc))
814	{
815	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
816	return VINF_SUCCESS;
817	}
818	return rc;
819	}
820	return VERR_VMX_INVALID_VMCS_LAUNCH_STATE;
821	}
822
823
824	/**
825	* Clears the VMCS specified by the VMCS info. object.
826	*
827	* @returns VBox status code.
828	* @param pVmcsInfo The VMCS info. object.
829	*/
830	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
831	{
832	Assert(pVmcsInfo);
833	Assert(pVmcsInfo->HCPhysVmcs);
834	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
835
836	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
837	if (RT_SUCCESS(rc))
838	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
839	return rc;
840	}
841
842
843	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
844	/**
845	* Switches the current VMCS to the one specified.
846	*
847	* @returns VBox status code.
848	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
849	* @param pVmcsInfoTo The VMCS info. object we are switching to.
850	*
851	* @remarks Called with interrupts disabled.
852	*/
853	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
854	{
855	Assert(pVmcsInfoFrom);
856	Assert(pVmcsInfoTo);
857
858	/*
859	* Clear the VMCS we are switching out if it has not already been cleared.
860	* This will sync any CPU internal data back to the VMCS.
861	*/
862	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
863	{
864	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
865	if (RT_SUCCESS(rc))
866	{ /* likely */ }
867	else
868	return rc;
869	}
870
871	/*
872	* Clear the VMCS we are switching to if it has not already been cleared.
873	* This will initialize the VMCS launch state to "clear" required for loading it.
874	*
875	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
876	*/
877	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
878	{
879	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
880	if (RT_SUCCESS(rc))
881	{ /* likely */ }
882	else
883	return rc;
884	}
885
886	/*
887	* Finally, load the VMCS we are switching to.
888	*/
889	return hmR0VmxLoadVmcs(pVmcsInfoTo);
890	}
891	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
892
893
894	/**
895	* Updates the VM's last error record.
896	*
897	* If there was a VMX instruction error, reads the error data from the VMCS and
898	* updates VCPU's last error record as well.
899	*
900	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
901	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
902	* VERR_VMX_INVALID_VMCS_FIELD.
903	* @param rc The error code.
904	*/
905	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
906	{
907	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
908	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
909	{
910	AssertPtrReturnVoid(pVCpu);
911	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
912	}
913	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
914	}
915
916
917	/**
918	* Reads the VM-entry interruption-information field from the VMCS into the VMX
919	* transient structure.
920	*
921	* @returns VBox status code.
922	* @param pVmxTransient The VMX-transient structure.
923	*
924	* @remarks No-long-jump zone!!!
925	*/
926	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
927	{
928	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
929	AssertRCReturn(rc, rc);
930	return VINF_SUCCESS;
931	}
932
933
934	#ifdef VBOX_STRICT
935	/**
936	* Reads the VM-entry exception error code field from the VMCS into
937	* the VMX transient structure.
938	*
939	* @returns VBox status code.
940	* @param pVmxTransient The VMX-transient structure.
941	*
942	* @remarks No-long-jump zone!!!
943	*/
944	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
945	{
946	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
947	AssertRCReturn(rc, rc);
948	return VINF_SUCCESS;
949	}
950
951
952	/**
953	* Reads the VM-entry exception error code field from the VMCS into
954	* the VMX transient structure.
955	*
956	* @returns VBox status code.
957	* @param pVmxTransient The VMX-transient structure.
958	*
959	* @remarks No-long-jump zone!!!
960	*/
961	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
962	{
963	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
964	AssertRCReturn(rc, rc);
965	return VINF_SUCCESS;
966	}
967	#endif /* VBOX_STRICT */
968
969
970	/**
971	* Reads the VM-exit interruption-information field from the VMCS into the VMX
972	* transient structure.
973	*
974	* @returns VBox status code.
975	* @param pVmxTransient The VMX-transient structure.
976	*/
977	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
978	{
979	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
980	{
981	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
982	AssertRCReturn(rc,rc);
983	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
984	}
985	return VINF_SUCCESS;
986	}
987
988
989	/**
990	* Reads the VM-exit interruption error code from the VMCS into the VMX
991	* transient structure.
992	*
993	* @returns VBox status code.
994	* @param pVmxTransient The VMX-transient structure.
995	*/
996	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
997	{
998	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
999	{
1000	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1001	AssertRCReturn(rc, rc);
1002	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1003	}
1004	return VINF_SUCCESS;
1005	}
1006
1007
1008	/**
1009	* Reads the VM-exit instruction length field from the VMCS into the VMX
1010	* transient structure.
1011	*
1012	* @returns VBox status code.
1013	* @param pVmxTransient The VMX-transient structure.
1014	*/
1015	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1016	{
1017	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1018	{
1019	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1020	AssertRCReturn(rc, rc);
1021	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1022	}
1023	return VINF_SUCCESS;
1024	}
1025
1026
1027	/**
1028	* Reads the VM-exit instruction-information field from the VMCS into
1029	* the VMX transient structure.
1030	*
1031	* @returns VBox status code.
1032	* @param pVmxTransient The VMX-transient structure.
1033	*/
1034	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1035	{
1036	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1037	{
1038	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1039	AssertRCReturn(rc, rc);
1040	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1041	}
1042	return VINF_SUCCESS;
1043	}
1044
1045
1046	/**
1047	* Reads the VM-exit Qualification from the VMCS into the VMX transient structure.
1048	*
1049	* @returns VBox status code.
1050	* @param pVCpu The cross context virtual CPU structure of the
1051	* calling EMT. (Required for the VMCS cache case.)
1052	* @param pVmxTransient The VMX-transient structure.
1053	*/
1054	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1055	{
1056	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1057	{
1058	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1059	AssertRCReturn(rc, rc);
1060	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1061	}
1062	return VINF_SUCCESS;
1063	}
1064
1065
1066	/**
1067	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1068	*
1069	* @returns VBox status code.
1070	* @param pVCpu The cross context virtual CPU structure of the
1071	* calling EMT. (Required for the VMCS cache case.)
1072	* @param pVmxTransient The VMX-transient structure.
1073	*/
1074	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1075	{
1076	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1077	{
1078	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1079	AssertRCReturn(rc, rc);
1080	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1081	}
1082	return VINF_SUCCESS;
1083	}
1084
1085
1086	/**
1087	* Reads the IDT-vectoring information field from the VMCS into the VMX
1088	* transient structure.
1089	*
1090	* @returns VBox status code.
1091	* @param pVmxTransient The VMX-transient structure.
1092	*
1093	* @remarks No-long-jump zone!!!
1094	*/
1095	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1096	{
1097	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1098	{
1099	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1100	AssertRCReturn(rc, rc);
1101	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1102	}
1103	return VINF_SUCCESS;
1104	}
1105
1106
1107	/**
1108	* Reads the IDT-vectoring error code from the VMCS into the VMX
1109	* transient structure.
1110	*
1111	* @returns VBox status code.
1112	* @param pVmxTransient The VMX-transient structure.
1113	*/
1114	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1115	{
1116	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1117	{
1118	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1119	AssertRCReturn(rc, rc);
1120	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1121	}
1122	return VINF_SUCCESS;
1123	}
1124
1125
1126	/**
1127	* Enters VMX root mode operation on the current CPU.
1128	*
1129	* @returns VBox status code.
1130	* @param pVM The cross context VM structure. Can be
1131	* NULL, after a resume.
1132	* @param HCPhysCpuPage Physical address of the VMXON region.
1133	* @param pvCpuPage Pointer to the VMXON region.
1134	*/
1135	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1136	{
1137	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1138	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1139	Assert(pvCpuPage);
1140	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1141
1142	if (pVM)
1143	{
1144	/* Write the VMCS revision identifier to the VMXON region. */
1145	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1146	}
1147
1148	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1149	RTCCUINTREG fEFlags = ASMIntDisableFlags();
1150
1151	/* Enable the VMX bit in CR4 if necessary. */
1152	RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1153
1154	/* Enter VMX root mode. */
1155	int rc = VMXEnable(HCPhysCpuPage);
1156	if (RT_FAILURE(rc))
1157	{
1158	if (!(uOldCr4 & X86_CR4_VMXE))
1159	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
1160
1161	if (pVM)
1162	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1163	}
1164
1165	/* Restore interrupts. */
1166	ASMSetFlags(fEFlags);
1167	return rc;
1168	}
1169
1170
1171	/**
1172	* Exits VMX root mode operation on the current CPU.
1173	*
1174	* @returns VBox status code.
1175	*/
1176	static int hmR0VmxLeaveRootMode(void)
1177	{
1178	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1179
1180	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1181	RTCCUINTREG fEFlags = ASMIntDisableFlags();
1182
1183	/* If we're for some reason not in VMX root mode, then don't leave it. */
1184	RTCCUINTREG uHostCR4 = ASMGetCR4();
1185
1186	int rc;
1187	if (uHostCR4 & X86_CR4_VMXE)
1188	{
1189	/* Exit VMX root mode and clear the VMX bit in CR4. */
1190	VMXDisable();
1191	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
1192	rc = VINF_SUCCESS;
1193	}
1194	else
1195	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1196
1197	/* Restore interrupts. */
1198	ASMSetFlags(fEFlags);
1199	return rc;
1200	}
1201
1202
1203	/**
1204	* Allocates and maps a physically contiguous page. The allocated page is
1205	* zero'd out (used by various VT-x structures).
1206	*
1207	* @returns IPRT status code.
1208	* @param pMemObj Pointer to the ring-0 memory object.
1209	* @param ppVirt Where to store the virtual address of the
1210	* allocation.
1211	* @param pHCPhys Where to store the physical address of the
1212	* allocation.
1213	*/
1214	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1215	{
1216	AssertPtr(pMemObj);
1217	AssertPtr(ppVirt);
1218	AssertPtr(pHCPhys);
1219	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1220	if (RT_FAILURE(rc))
1221	return rc;
1222	ppVirt = RTR0MemObjAddress(pMemObj);
1223	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1224	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1225	return VINF_SUCCESS;
1226	}
1227
1228
1229	/**
1230	* Frees and unmaps an allocated, physical page.
1231	*
1232	* @param pMemObj Pointer to the ring-0 memory object.
1233	* @param ppVirt Where to re-initialize the virtual address of
1234	* allocation as 0.
1235	* @param pHCPhys Where to re-initialize the physical address of the
1236	* allocation as 0.
1237	*/
1238	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1239	{
1240	AssertPtr(pMemObj);
1241	AssertPtr(ppVirt);
1242	AssertPtr(pHCPhys);
1243	/* NULL is valid, accepted and ignored by the free function below. */
1244	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1245	*pMemObj = NIL_RTR0MEMOBJ;
1246	*ppVirt = NULL;
1247	*pHCPhys = NIL_RTHCPHYS;
1248	}
1249
1250
1251	/**
1252	* Initializes a VMCS info. object.
1253	*
1254	* @param pVmcsInfo The VMCS info. object.
1255	*/
1256	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1257	{
1258	RT_ZERO(*pVmcsInfo);
1259
1260	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1261	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1262	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1263	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1264	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1265	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1266	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1267	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1268	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1269	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1270	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1271	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1272	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1273	}
1274
1275
1276	/**
1277	* Frees the VT-x structures for a VMCS info. object.
1278	*
1279	* @param pVM The cross context VM structure.
1280	* @param pVmcsInfo The VMCS info. object.
1281	*/
1282	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1283	{
1284	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1285
1286	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1287	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1288
1289	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1290	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1291	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1292
1293	hmR0VmxInitVmcsInfo(pVmcsInfo);
1294	}
1295
1296
1297	/**
1298	* Allocates the VT-x structures for a VMCS info. object.
1299	*
1300	* @returns VBox status code.
1301	* @param pVCpu The cross context virtual CPU structure.
1302	* @param pVmcsInfo The VMCS info. object.
1303	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1304	*/
1305	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1306	{
1307	PVM pVM = pVCpu->CTX_SUFF(pVM);
1308
1309	/* Allocate the guest VM control structure (VMCS). */
1310	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1311	if (RT_SUCCESS(rc))
1312	{
1313	if (!fIsNstGstVmcs)
1314	{
1315	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1316	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1317	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1318	{
1319	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1320	NULL /* pR3Ptr /, NULL / pRCPtr */);
1321	}
1322	}
1323	else
1324	{
1325	Assert(pVmcsInfo->HCPhysVirtApic == NIL_RTHCPHYS);
1326	Assert(!pVmcsInfo->pbVirtApic);
1327	}
1328
1329	if (RT_SUCCESS(rc))
1330	{
1331	/*
1332	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1333	* transparent accesses of specific MSRs.
1334	*
1335	* If the condition for enabling MSR bitmaps changes here, don't forget to
1336	* update HMIsMsrBitmapActive().
1337	*
1338	* We don't share MSR bitmaps between the guest and nested-guest as we then
1339	* don't need to care about carefully restoring the guest MSR bitmap.
1340	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1341	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1342	*/
1343	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1344	{
1345	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1346	if (RT_SUCCESS(rc))
1347	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1348	}
1349
1350	if (RT_SUCCESS(rc))
1351	{
1352	/*
1353	* Allocate the VM-entry MSR-load area for the guest MSRs.
1354	*
1355	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1356	* the guest and nested-guest.
1357	*/
1358	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1359	&pVmcsInfo->HCPhysGuestMsrLoad);
1360	if (RT_SUCCESS(rc))
1361	{
1362	/*
1363	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1364	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1365	*/
1366	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1367	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1368	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1369
1370	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1371	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1372	&pVmcsInfo->HCPhysHostMsrLoad);
1373	}
1374	}
1375	}
1376	}
1377
1378	return rc;
1379	}
1380
1381
1382	/**
1383	* Free all VT-x structures for the VM.
1384	*
1385	* @returns IPRT status code.
1386	* @param pVM The cross context VM structure.
1387	*/
1388	static void hmR0VmxStructsFree(PVM pVM)
1389	{
1390	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1391	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1392	#endif
1393	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1394
1395	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1396	{
1397	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1398	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1399	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1400	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1401	if (pVM->cpum.ro.GuestFeatures.fVmx)
1402	{
1403	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1404	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1405	}
1406	#endif
1407	}
1408	}
1409
1410
1411	/**
1412	* Allocate all VT-x structures for the VM.
1413	*
1414	* @returns IPRT status code.
1415	* @param pVM The cross context VM structure.
1416	*/
1417	static int hmR0VmxStructsAlloc(PVM pVM)
1418	{
1419	/*
1420	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1421	* The VMCS size cannot be more than 4096 bytes.
1422	*
1423	* See Intel spec. Appendix A.1 "Basic VMX Information".
1424	*/
1425	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1426	if (cbVmcs <= X86_PAGE_4K_SIZE)
1427	{ /* likely */ }
1428	else
1429	{
1430	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1431	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1432	}
1433
1434	/*
1435	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1436	*/
1437	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1438	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1439	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1440	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1441	#endif
1442
1443	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1444	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1445	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1446
1447	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1448	{
1449	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1450	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1451	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1452	}
1453
1454	/*
1455	* Allocate per-VM VT-x structures.
1456	*/
1457	int rc = VINF_SUCCESS;
1458	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1459	/* Allocate crash-dump magic scratch page. */
1460	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1461	if (RT_FAILURE(rc))
1462	{
1463	hmR0VmxStructsFree(pVM);
1464	return rc;
1465	}
1466	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1467	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1468	#endif
1469
1470	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1471	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1472	{
1473	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1474	&pVM->hm.s.vmx.HCPhysApicAccess);
1475	if (RT_FAILURE(rc))
1476	{
1477	hmR0VmxStructsFree(pVM);
1478	return rc;
1479	}
1480	}
1481
1482	/*
1483	* Initialize per-VCPU VT-x structures.
1484	*/
1485	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1486	{
1487	/* Allocate the guest VMCS structures. */
1488	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1489	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
1490	if (RT_SUCCESS(rc))
1491	{
1492	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1493	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
1494	if (pVM->cpum.ro.GuestFeatures.fVmx)
1495	{
1496	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
1497	if (RT_SUCCESS(rc))
1498	{ /* likely */ }
1499	else
1500	break;
1501	}
1502	#endif
1503	}
1504	else
1505	break;
1506	}
1507
1508	if (RT_FAILURE(rc))
1509	{
1510	hmR0VmxStructsFree(pVM);
1511	return rc;
1512	}
1513
1514	return VINF_SUCCESS;
1515	}
1516
1517
1518	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1519	/**
1520	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
1521	*
1522	* @returns @c true if the MSR is intercepted, @c false otherwise.
1523	* @param pvMsrBitmap The MSR bitmap.
1524	* @param offMsr The MSR byte offset.
1525	* @param iBit The bit offset from the byte offset.
1526	*/
1527	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
1528	{
1529	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
1530	Assert(pbMsrBitmap);
1531	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
1532	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
1533	}
1534	#endif
1535
1536
1537	/**
1538	* Sets the permission bits for the specified MSR in the given MSR bitmap.
1539	*
1540	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
1541	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
1542	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
1543	* the read/write access of this MSR.
1544	*
1545	* @param pVmcsInfo The VMCS info. object.
1546	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1547	* @param idMsr The MSR value.
1548	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
1549	* include both a read -and- a write permission!
1550	*
1551	* @sa HMGetVmxMsrPermission.
1552	*/
1553	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
1554	{
1555	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
1556	Assert(pbMsrBitmap);
1557	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
1558
1559	/*
1560	* MSR-bitmap Layout:
1561	* Byte index MSR range Interpreted as
1562	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1563	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1564	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1565	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1566	*
1567	* A bit corresponding to an MSR within the above range causes a VM-exit
1568	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
1569	* the MSR range, it always cause a VM-exit.
1570	*
1571	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
1572	*/
1573	uint16_t const offBitmapRead = 0;
1574	uint16_t const offBitmapWrite = 0x800;
1575	uint16_t offMsr;
1576	int32_t iBit;
1577	if (idMsr <= UINT32_C(0x00001fff))
1578	{
1579	offMsr = 0;
1580	iBit = idMsr;
1581	}
1582	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1583	{
1584	offMsr = 0x400;
1585	iBit = idMsr - UINT32_C(0xc0000000);
1586	}
1587	else
1588	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
1589
1590	/*
1591	* Set the MSR read permission.
1592	*/
1593	uint16_t const offMsrRead = offBitmapRead + offMsr;
1594	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
1595	if (fMsrpm & VMXMSRPM_ALLOW_RD)
1596	{
1597	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1598	bool const fClear = !fIsNstGstVmcs ? true
1599	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
1600	#else
1601	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1602	bool const fClear = true;
1603	#endif
1604	if (fClear)
1605	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
1606	}
1607	else
1608	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
1609
1610	/*
1611	* Set the MSR write permission.
1612	*/
1613	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
1614	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
1615	if (fMsrpm & VMXMSRPM_ALLOW_WR)
1616	{
1617	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1618	bool const fClear = !fIsNstGstVmcs ? true
1619	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
1620	#else
1621	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1622	bool const fClear = true;
1623	#endif
1624	if (fClear)
1625	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
1626	}
1627	else
1628	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
1629	}
1630
1631
1632	/**
1633	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1634	* area.
1635	*
1636	* @returns VBox status code.
1637	* @param pVCpu The cross context virtual CPU structure.
1638	* @param pVmcsInfo The VMCS info. object.
1639	* @param cMsrs The number of MSRs.
1640	*/
1641	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
1642	{
1643	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1644	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1645	if (RT_UNLIKELY(cMsrs >= cMaxSupportedMsrs))
1646	{
1647	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1648	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1649	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1650	}
1651
1652	/* Commit the MSR counts to the VMCS and update the cache. */
1653	int rc = VINF_SUCCESS;
1654	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
1655	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1656	if (pVmcsInfo->cExitMsrStore != cMsrs)
1657	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1658	if (pVmcsInfo->cExitMsrLoad != cMsrs)
1659	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1660	AssertRCReturn(rc, rc);
1661
1662	pVmcsInfo->cEntryMsrLoad = cMsrs;
1663	pVmcsInfo->cExitMsrStore = cMsrs;
1664	pVmcsInfo->cExitMsrLoad = cMsrs;
1665
1666	return VINF_SUCCESS;
1667	}
1668
1669
1670	/**
1671	* Adds a new (or updates the value of an existing) guest/host MSR
1672	* pair to be swapped during the world-switch as part of the
1673	* auto-load/store MSR area in the VMCS.
1674	*
1675	* @returns VBox status code.
1676	* @param pVCpu The cross context virtual CPU structure.
1677	* @param pVmxTransient The VMX-transient structure.
1678	* @param idMsr The MSR.
1679	* @param uGuestMsrValue Value of the guest MSR.
1680	* @param fSetReadWrite Whether to set the guest read/write access of this
1681	* MSR (thus not causing a VM-exit).
1682	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1683	* necessary.
1684	*/
1685	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
1686	bool fSetReadWrite, bool fUpdateHostMsr)
1687	{
1688	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1689	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1690	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1691	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1692	uint32_t i;
1693
1694	/* Paranoia. */
1695	Assert(pGuestMsrLoad);
1696
1697	/* Check if the MSR already exists in the VM-entry MSR-load area. */
1698	for (i = 0; i < cMsrs; i++)
1699	{
1700	if (pGuestMsrLoad->u32Msr == idMsr)
1701	break;
1702	pGuestMsrLoad++;
1703	}
1704
1705	bool fAdded = false;
1706	if (i == cMsrs)
1707	{
1708	/* The MSR does not exist, bump the MSR coun to make room for the new MSR. */
1709	++cMsrs;
1710	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1711	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
1712
1713	/* Set the guest to read/write this MSR without causing VM-exits. */
1714	if ( fSetReadWrite
1715	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
1716	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
1717
1718	fAdded = true;
1719	}
1720
1721	/* Update the MSR value for the newly added or already existing MSR. */
1722	pGuestMsrLoad->u32Msr = idMsr;
1723	pGuestMsrLoad->u64Value = uGuestMsrValue;
1724
1725	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
1726	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1727	{
1728	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1729	pGuestMsrStore += i;
1730	pGuestMsrStore->u32Msr = idMsr;
1731	pGuestMsrStore->u64Value = 0;
1732	}
1733
1734	/* Update the corresponding slot in the host MSR area. */
1735	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1736	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad && pHostMsr != pVmcsInfo->pvGuestMsrStore);
1737	pHostMsr += i;
1738	pHostMsr->u32Msr = idMsr;
1739
1740	/*
1741	* Only if the caller requests to update the host MSR value AND we've newly added the
1742	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
1743	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
1744	*
1745	* We do this for performance reasons since reading MSRs may be quite expensive.
1746	*/
1747	if ( fAdded
1748	&& fUpdateHostMsr)
1749	{
1750	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1751	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1752	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1753	}
1754	return VINF_SUCCESS;
1755	}
1756
1757
1758	/**
1759	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1760	* auto-load/store MSR area in the VMCS.
1761	*
1762	* @returns VBox status code.
1763	* @param pVCpu The cross context virtual CPU structure.
1764	* @param pVmxTransient The VMX-transient structure.
1765	* @param idMsr The MSR.
1766	*/
1767	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
1768	{
1769	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1770	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1771	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1772	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1773
1774	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
1775	for (uint32_t i = 0; i < cMsrs; i++)
1776	{
1777	/* Find the MSR. */
1778	if (pGuestMsrLoad->u32Msr == idMsr)
1779	{
1780	/* If it's the last MSR, simply reduce the count. */
1781	if (i == cMsrs - 1)
1782	{
1783	--cMsrs;
1784	break;
1785	}
1786
1787	/* Remove it by copying the last MSR in place of it, and reducing the count. */
1788	PVMXAUTOMSR pLastGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1789	pLastGuestMsrLoad += cMsrs - 1;
1790	pGuestMsrLoad->u32Msr = pLastGuestMsrLoad->u32Msr;
1791	pGuestMsrLoad->u64Value = pLastGuestMsrLoad->u64Value;
1792
1793	/* Remove it from the VM-exit MSR-store area if we are using a different page. */
1794	if (fSeparateExitMsrStorePage)
1795	{
1796	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1797	PVMXAUTOMSR pLastGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1798	pGuestMsrStore += i;
1799	pLastGuestMsrStore += cMsrs - 1;
1800	Assert(pGuestMsrStore->u32Msr == idMsr);
1801	pGuestMsrStore->u32Msr = pLastGuestMsrStore->u32Msr;
1802	pGuestMsrStore->u64Value = pLastGuestMsrStore->u64Value;
1803	}
1804
1805	/* Remove it from the VM-exit MSR-load area. */
1806	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1807	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1808	pHostMsr += i;
1809	pLastHostMsr += cMsrs - 1;
1810	Assert(pHostMsr->u32Msr == idMsr);
1811	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1812	pHostMsr->u64Value = pLastHostMsr->u64Value;
1813	--cMsrs;
1814	break;
1815	}
1816	pGuestMsrLoad++;
1817	}
1818
1819	/* Update the VMCS if the count changed (meaning the MSR was found). */
1820	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
1821	{
1822	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1823	AssertRCReturn(rc, rc);
1824
1825	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1826	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1827	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
1828
1829	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
1830	return VINF_SUCCESS;
1831	}
1832
1833	return VERR_NOT_FOUND;
1834	}
1835
1836
1837	/**
1838	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
1839	*
1840	* @returns @c true if found, @c false otherwise.
1841	* @param pVmcsInfo The VMCS info. object.
1842	* @param idMsr The MSR to find.
1843	*/
1844	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
1845	{
1846	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1847	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
1848	for (uint32_t i = 0; i < cMsrs; i++)
1849	{
1850	if (pGuestMsrLoad->u32Msr == idMsr)
1851	return true;
1852	pGuestMsrLoad++;
1853	}
1854	return false;
1855	}
1856
1857
1858	/**
1859	* Updates the value of all host MSRs in the VM-exit MSR-load area.
1860	*
1861	* @param pVCpu The cross context virtual CPU structure.
1862	* @param pVmcsInfo The VMCS info. object.
1863	*
1864	* @remarks No-long-jump zone!!!
1865	*/
1866	static void hmR0VmxUpdateAutoLoadHostMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
1867	{
1868	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1869	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
1870
1871	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1872	Assert(pHostMsrLoad);
1873
1874	for (uint32_t i = 0; i < cMsrs; i++, pHostMsrLoad++)
1875	{
1876	/*
1877	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1878	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1879	*/
1880	if (pHostMsrLoad->u32Msr == MSR_K6_EFER)
1881	pHostMsrLoad->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
1882	else
1883	pHostMsrLoad->u64Value = ASMRdMsr(pHostMsrLoad->u32Msr);
1884	}
1885	}
1886
1887
1888	/**
1889	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1890	* perform lazy restoration of the host MSRs while leaving VT-x.
1891	*
1892	* @param pVCpu The cross context virtual CPU structure.
1893	*
1894	* @remarks No-long-jump zone!!!
1895	*/
1896	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1897	{
1898	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1899
1900	/*
1901	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
1902	*/
1903	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1904	{
1905	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1906	#if HC_ARCH_BITS == 64
1907	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1908	{
1909	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
1910	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
1911	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
1912	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1913	}
1914	#endif
1915	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1916	}
1917	}
1918
1919
1920	/**
1921	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1922	* lazily while leaving VT-x.
1923	*
1924	* @returns true if it does, false otherwise.
1925	* @param pVCpu The cross context virtual CPU structure.
1926	* @param idMsr The MSR to check.
1927	*/
1928	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t idMsr)
1929	{
1930	NOREF(pVCpu);
1931	#if HC_ARCH_BITS == 64
1932	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1933	{
1934	switch (idMsr)
1935	{
1936	case MSR_K8_LSTAR:
1937	case MSR_K6_STAR:
1938	case MSR_K8_SF_MASK:
1939	case MSR_K8_KERNEL_GS_BASE:
1940	return true;
1941	}
1942	}
1943	#else
1944	RT_NOREF(pVCpu, idMsr);
1945	#endif
1946	return false;
1947	}
1948
1949
1950	/**
1951	* Loads a set of guests MSRs to allow read/passthru to the guest.
1952	*
1953	* The name of this function is slightly confusing. This function does NOT
1954	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1955	* common prefix for functions dealing with "lazy restoration" of the shared
1956	* MSRs.
1957	*
1958	* @param pVCpu The cross context virtual CPU structure.
1959	*
1960	* @remarks No-long-jump zone!!!
1961	*/
1962	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1963	{
1964	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1965	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1966
1967	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1968	#if HC_ARCH_BITS == 64
1969	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1970	{
1971	/*
1972	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1973	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1974	* we can skip a few MSR writes.
1975	*
1976	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1977	* guest MSR values in the guest-CPU context might be different to what's currently
1978	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1979	* CPU, see @bugref{8728}.
1980	*/
1981	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1982	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1983	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
1984	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
1985	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
1986	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
1987	{
1988	#ifdef VBOX_STRICT
1989	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1990	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1991	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1992	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1993	#endif
1994	}
1995	else
1996	{
1997	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1998	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
1999	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2000	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2001	}
2002	}
2003	#endif
2004	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2005	}
2006
2007
2008	/**
2009	* Performs lazy restoration of the set of host MSRs if they were previously
2010	* loaded with guest MSR values.
2011	*
2012	* @param pVCpu The cross context virtual CPU structure.
2013	*
2014	* @remarks No-long-jump zone!!!
2015	* @remarks The guest MSRs should have been saved back into the guest-CPU
2016	* context by hmR0VmxImportGuestState()!!!
2017	*/
2018	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2019	{
2020	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2021	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2022
2023	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2024	{
2025	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2026	#if HC_ARCH_BITS == 64
2027	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2028	{
2029	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2030	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2031	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2032	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2033	}
2034	#endif
2035	}
2036	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2037	}
2038
2039
2040	/**
2041	* Verifies that our cached values of the VMCS fields are all consistent with
2042	* what's actually present in the VMCS.
2043	*
2044	* @returns VBox status code.
2045	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2046	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2047	* VMCS content. HMCPU error-field is
2048	* updated, see VMX_VCI_XXX.
2049	* @param pVCpu The cross context virtual CPU structure.
2050	* @param pVmcsInfo The VMCS info. object.
2051	*/
2052	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2053	{
2054	uint32_t u32Val;
2055	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2056	AssertRCReturn(rc, rc);
2057	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2058	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32EntryCtls, u32Val),
2059	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2060	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2061
2062	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2063	AssertRCReturn(rc, rc);
2064	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2065	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ExitCtls, u32Val),
2066	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2067	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2068
2069	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2070	AssertRCReturn(rc, rc);
2071	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2072	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32PinCtls, u32Val),
2073	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2074	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2075
2076	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2077	AssertRCReturn(rc, rc);
2078	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2079	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls, u32Val),
2080	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2081	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2082
2083	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2084	{
2085	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2086	AssertRCReturn(rc, rc);
2087	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2088	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls2, u32Val),
2089	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2090	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2091	}
2092
2093	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2094	AssertRCReturn(rc, rc);
2095	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2096	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32XcptBitmap, u32Val),
2097	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2098	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2099
2100	uint64_t u64Val;
2101	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2102	AssertRCReturn(rc, rc);
2103	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2104	("Cache=%#RX64 VMCS=%#RX64\n", pVmcsInfo->u64TscOffset, u64Val),
2105	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2106	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2107
2108	return VINF_SUCCESS;
2109	}
2110
2111
2112	#ifdef VBOX_STRICT
2113	/**
2114	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2115	*
2116	* @param pVCpu The cross context virtual CPU structure.
2117	* @param pVmcsInfo The VMCS info. object.
2118	*/
2119	static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2120	{
2121	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2122
2123	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2124	{
2125	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2126	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2127	uint64_t uVmcsEferMsrVmcs;
2128	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2129	AssertRC(rc);
2130
2131	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2132	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2133	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2134	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2135	}
2136	}
2137
2138
2139	/**
2140	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2141	* VMCS are correct.
2142	*
2143	* @param pVCpu The cross context virtual CPU structure.
2144	*/
2145	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2146	{
2147	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2148
2149	/* Verify MSR counts in the VMCS are what we think it should be. */
2150	uint32_t cMsrs;
2151	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs);
2152	AssertRC(rc);
2153	Assert(cMsrs == pVmcsInfo->cEntryMsrLoad);
2154
2155	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs);
2156	AssertRC(rc);
2157	Assert(cMsrs == pVmcsInfo->cExitMsrStore);
2158
2159	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs);
2160	AssertRC(rc);
2161	Assert(cMsrs == pVmcsInfo->cExitMsrLoad);
2162
2163	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2164	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2165
2166	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2167	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2168	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2169	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2170	for (uint32_t i = 0; i < cMsrs; i++)
2171	{
2172	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2173	if (fSeparateExitMsrStorePage)
2174	{
2175	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2176	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2177	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2178	}
2179
2180	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2181	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2182	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2183
2184	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2185	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2186	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2187	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2188
2189	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2190	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2191	{
2192	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2193	if (pGuestMsrLoad->u32Msr == MSR_K6_EFER)
2194	{
2195	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2196	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2197	}
2198	else
2199	{
2200	AssertMsgReturnVoid((fMsrpm & (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR))
2201	== (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR),
2202	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2203	}
2204	}
2205
2206	/* Move to the next MSR. */
2207	pHostMsrLoad++;
2208	pGuestMsrLoad++;
2209	pGuestMsrStore++;
2210	}
2211	}
2212	#endif /* VBOX_STRICT */
2213
2214
2215	/**
2216	* Flushes the TLB using EPT.
2217	*
2218	* @returns VBox status code.
2219	* @param pVCpu The cross context virtual CPU structure of the calling
2220	* EMT. Can be NULL depending on @a enmTlbFlush.
2221	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2222	* enmTlbFlush.
2223	* @param enmTlbFlush Type of flush.
2224	*
2225	* @remarks Caller is responsible for making sure this function is called only
2226	* when NestedPaging is supported and providing @a enmTlbFlush that is
2227	* supported by the CPU.
2228	* @remarks Can be called with interrupts disabled.
2229	*/
2230	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2231	{
2232	uint64_t au64Descriptor[2];
2233	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2234	au64Descriptor[0] = 0;
2235	else
2236	{
2237	Assert(pVCpu);
2238	Assert(pVmcsInfo);
2239	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2240	}
2241	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2242
2243	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2244	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2245
2246	if ( RT_SUCCESS(rc)
2247	&& pVCpu)
2248	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2249	}
2250
2251
2252	/**
2253	* Flushes the TLB using VPID.
2254	*
2255	* @returns VBox status code.
2256	* @param pVCpu The cross context virtual CPU structure of the calling
2257	* EMT. Can be NULL depending on @a enmTlbFlush.
2258	* @param enmTlbFlush Type of flush.
2259	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2260	* on @a enmTlbFlush).
2261	*
2262	* @remarks Can be called with interrupts disabled.
2263	*/
2264	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2265	{
2266	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2267
2268	uint64_t au64Descriptor[2];
2269	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2270	{
2271	au64Descriptor[0] = 0;
2272	au64Descriptor[1] = 0;
2273	}
2274	else
2275	{
2276	AssertPtr(pVCpu);
2277	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2278	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2279	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2280	au64Descriptor[1] = GCPtr;
2281	}
2282
2283	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2284	AssertMsg(rc == VINF_SUCCESS,
2285	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2286
2287	if ( RT_SUCCESS(rc)
2288	&& pVCpu)
2289	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2290	NOREF(rc);
2291	}
2292
2293
2294	/**
2295	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2296	* otherwise there is nothing really to invalidate.
2297	*
2298	* @returns VBox status code.
2299	* @param pVCpu The cross context virtual CPU structure.
2300	* @param GCVirt Guest virtual address of the page to invalidate.
2301	*/
2302	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2303	{
2304	AssertPtr(pVCpu);
2305	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2306
2307	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2308	{
2309	/*
2310	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2311	* the EPT case. See @bugref{6043} and @bugref{6177}.
2312	*
2313	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2314	* as this function maybe called in a loop with individual addresses.
2315	*/
2316	PVM pVM = pVCpu->CTX_SUFF(pVM);
2317	if (pVM->hm.s.vmx.fVpid)
2318	{
2319	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2320
2321	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2322	/*
2323	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2324	* where executing INVVPID outside 64-bit mode does not flush translations of
2325	* 64-bit linear addresses, see @bugref{6208#c72}.
2326	*/
2327	if (RT_HI_U32(GCVirt))
2328	fVpidFlush = false;
2329	#endif
2330
2331	if (fVpidFlush)
2332	{
2333	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2334	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2335	}
2336	else
2337	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2338	}
2339	else if (pVM->hm.s.fNestedPaging)
2340	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2341	}
2342
2343	return VINF_SUCCESS;
2344	}
2345
2346
2347	/**
2348	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2349	* case where neither EPT nor VPID is supported by the CPU.
2350	*
2351	* @param pHostCpu The HM physical-CPU structure.
2352	* @param pVCpu The cross context virtual CPU structure.
2353	*
2354	* @remarks Called with interrupts disabled.
2355	*/
2356	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2357	{
2358	AssertPtr(pVCpu);
2359	AssertPtr(pHostCpu);
2360
2361	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2362
2363	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2364	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2365	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2366	pVCpu->hm.s.fForceTLBFlush = false;
2367	return;
2368	}
2369
2370
2371	/**
2372	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2373	*
2374	* @param pHostCpu The HM physical-CPU structure.
2375	* @param pVCpu The cross context virtual CPU structure.
2376	* @param pVmcsInfo The VMCS info. object.
2377	*
2378	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2379	* nomenclature. The reason is, to avoid confusion in compare statements
2380	* since the host-CPU copies are named "ASID".
2381	*
2382	* @remarks Called with interrupts disabled.
2383	*/
2384	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2385	{
2386	#ifdef VBOX_WITH_STATISTICS
2387	bool fTlbFlushed = false;
2388	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2389	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2390	if (!fTlbFlushed) \
2391	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2392	} while (0)
2393	#else
2394	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2395	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2396	#endif
2397
2398	AssertPtr(pVCpu);
2399	AssertPtr(pHostCpu);
2400	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2401
2402	PVM pVM = pVCpu->CTX_SUFF(pVM);
2403	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2404	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2405	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2406
2407	/*
2408	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2409	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2410	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2411	* cannot reuse the current ASID anymore.
2412	*/
2413	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2414	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2415	{
2416	++pHostCpu->uCurrentAsid;
2417	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2418	{
2419	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2420	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2421	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2422	}
2423
2424	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2425	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2426	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2427
2428	/*
2429	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2430	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2431	*/
2432	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2433	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2434	HMVMX_SET_TAGGED_TLB_FLUSHED();
2435	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2436	}
2437	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2438	{
2439	/*
2440	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2441	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2442	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2443	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2444	* mappings, see @bugref{6568}.
2445	*
2446	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2447	*/
2448	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2449	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2450	HMVMX_SET_TAGGED_TLB_FLUSHED();
2451	}
2452
2453	pVCpu->hm.s.fForceTLBFlush = false;
2454	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2455
2456	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
2457	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
2458	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2459	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2460	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2461	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2462	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2463	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2464	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2465
2466	/* Update VMCS with the VPID. */
2467	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2468	AssertRC(rc);
2469
2470	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2471	}
2472
2473
2474	/**
2475	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2476	*
2477	* @param pHostCpu The HM physical-CPU structure.
2478	* @param pVCpu The cross context virtual CPU structure.
2479	* @param pVmcsInfo The VMCS info. object.
2480	*
2481	* @remarks Called with interrupts disabled.
2482	*/
2483	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2484	{
2485	AssertPtr(pVCpu);
2486	AssertPtr(pHostCpu);
2487	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2488	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2489	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2490
2491	/*
2492	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2493	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2494	*/
2495	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2496	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2497	{
2498	pVCpu->hm.s.fForceTLBFlush = true;
2499	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2500	}
2501
2502	/* Check for explicit TLB flushes. */
2503	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2504	{
2505	pVCpu->hm.s.fForceTLBFlush = true;
2506	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2507	}
2508
2509	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2510	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2511
2512	if (pVCpu->hm.s.fForceTLBFlush)
2513	{
2514	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2515	pVCpu->hm.s.fForceTLBFlush = false;
2516	}
2517	}
2518
2519
2520	/**
2521	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2522	*
2523	* @param pHostCpu The HM physical-CPU structure.
2524	* @param pVCpu The cross context virtual CPU structure.
2525	*
2526	* @remarks Called with interrupts disabled.
2527	*/
2528	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2529	{
2530	AssertPtr(pVCpu);
2531	AssertPtr(pHostCpu);
2532	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2533	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2534	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2535
2536	/*
2537	* Force a TLB flush for the first world switch if the current CPU differs from the one we
2538	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2539	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2540	* cannot reuse the current ASID anymore.
2541	*/
2542	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2543	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2544	{
2545	pVCpu->hm.s.fForceTLBFlush = true;
2546	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2547	}
2548
2549	/* Check for explicit TLB flushes. */
2550	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2551	{
2552	/*
2553	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2554	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2555	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
2556	* include fExplicitFlush's too) - an obscure corner case.
2557	*/
2558	pVCpu->hm.s.fForceTLBFlush = true;
2559	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2560	}
2561
2562	PVM pVM = pVCpu->CTX_SUFF(pVM);
2563	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2564	if (pVCpu->hm.s.fForceTLBFlush)
2565	{
2566	++pHostCpu->uCurrentAsid;
2567	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2568	{
2569	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2570	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2571	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2572	}
2573
2574	pVCpu->hm.s.fForceTLBFlush = false;
2575	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2576	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2577	if (pHostCpu->fFlushAsidBeforeUse)
2578	{
2579	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2580	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2581	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2582	{
2583	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2584	pHostCpu->fFlushAsidBeforeUse = false;
2585	}
2586	else
2587	{
2588	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2589	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2590	}
2591	}
2592	}
2593
2594	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2595	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2596	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2597	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2598	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2599	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2600	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2601
2602	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2603	AssertRC(rc);
2604	}
2605
2606
2607	/**
2608	* Flushes the guest TLB entry based on CPU capabilities.
2609	*
2610	* @param pHostCpu The HM physical-CPU structure.
2611	* @param pVCpu The cross context virtual CPU structure.
2612	* @param pVmcsInfo The VMCS info. object.
2613	*
2614	* @remarks Called with interrupts disabled.
2615	*/
2616	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2617	{
2618	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2619	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2620	#endif
2621	PVM pVM = pVCpu->CTX_SUFF(pVM);
2622	switch (pVM->hm.s.vmx.enmTlbFlushType)
2623	{
2624	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
2625	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
2626	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
2627	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
2628	default:
2629	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2630	break;
2631	}
2632	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2633	}
2634
2635
2636	/**
2637	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2638	* TLB entries from the host TLB before VM-entry.
2639	*
2640	* @returns VBox status code.
2641	* @param pVM The cross context VM structure.
2642	*/
2643	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2644	{
2645	/*
2646	* Determine optimal flush type for nested paging.
2647	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
2648	* unrestricted guest execution (see hmR3InitFinalizeR0()).
2649	*/
2650	if (pVM->hm.s.fNestedPaging)
2651	{
2652	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2653	{
2654	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2655	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2656	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2657	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2658	else
2659	{
2660	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2661	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2662	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2663	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2664	}
2665
2666	/* Make sure the write-back cacheable memory type for EPT is supported. */
2667	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2668	{
2669	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2670	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2671	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2672	}
2673
2674	/* EPT requires a page-walk length of 4. */
2675	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2676	{
2677	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2678	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2679	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2680	}
2681	}
2682	else
2683	{
2684	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2685	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2686	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2687	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2688	}
2689	}
2690
2691	/*
2692	* Determine optimal flush type for VPID.
2693	*/
2694	if (pVM->hm.s.vmx.fVpid)
2695	{
2696	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2697	{
2698	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2699	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2700	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2701	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2702	else
2703	{
2704	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2705	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2706	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2707	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2708	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2709	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2710	pVM->hm.s.vmx.fVpid = false;
2711	}
2712	}
2713	else
2714	{
2715	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2716	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2717	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2718	pVM->hm.s.vmx.fVpid = false;
2719	}
2720	}
2721
2722	/*
2723	* Setup the handler for flushing tagged-TLBs.
2724	*/
2725	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2726	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2727	else if (pVM->hm.s.fNestedPaging)
2728	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2729	else if (pVM->hm.s.vmx.fVpid)
2730	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2731	else
2732	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2733	return VINF_SUCCESS;
2734	}
2735
2736
2737	/**
2738	* Sets up the virtual-APIC page address for the VMCS.
2739	*
2740	* @returns VBox status code.
2741	* @param pVmcsInfo The VMCS info. object.
2742	*/
2743	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PCVMXVMCSINFO pVmcsInfo)
2744	{
2745	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
2746	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
2747	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2748	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
2749	}
2750
2751
2752	/**
2753	* Sets up the MSR-bitmap address for the VMCS.
2754	*
2755	* @returns VBox status code.
2756	* @param pVmcsInfo The VMCS info. object.
2757	*/
2758	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PCVMXVMCSINFO pVmcsInfo)
2759	{
2760	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
2761	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
2762	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2763	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
2764	}
2765
2766
2767	/**
2768	* Sets up the APIC-access page address for the VMCS.
2769	*
2770	* @returns VBox status code.
2771	* @param pVCpu The cross context virtual CPU structure.
2772	*/
2773	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
2774	{
2775	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
2776	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
2777	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2778	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
2779	}
2780
2781
2782	/**
2783	* Sets up the VMCS link pointer for the VMCS.
2784	*
2785	* @returns VBox status code.
2786	* @param pVmcsInfo The VMCS info. object.
2787	*/
2788	DECLINLINE(int) hmR0VmxSetupVmcsLinkPtr(PVMXVMCSINFO pVmcsInfo)
2789	{
2790	uint64_t const u64VmcsLinkPtr = pVmcsInfo->u64VmcsLinkPtr;
2791	Assert(u64VmcsLinkPtr == UINT64_C(0xffffffffffffffff)); /* Bits 63:0 MB1. */
2792	return VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, u64VmcsLinkPtr);
2793	}
2794
2795
2796	/**
2797	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
2798	* in the VMCS.
2799	*
2800	* @returns VBox status code.
2801	* @param pVmcsInfo The VMCS info. object.
2802	*/
2803	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMXVMCSINFO pVmcsInfo)
2804	{
2805	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
2806	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
2807	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2808
2809	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
2810	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
2811	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
2812
2813	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
2814	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
2815	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2816
2817	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
2818	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
2819	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
2820	AssertRCReturn(rc, rc);
2821	return VINF_SUCCESS;
2822	}
2823
2824
2825	/**
2826	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
2827	*
2828	* @param pVCpu The cross context virtual CPU structure.
2829	* @param pVmcsInfo The VMCS info. object.
2830	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2831	*/
2832	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2833	{
2834	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
2835
2836	/*
2837	* The guest can access the following MSRs (read, write) without causing
2838	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
2839	*/
2840	PVM pVM = pVCpu->CTX_SUFF(pVM);
2841	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2842	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2843	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2844	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2845	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2846
2847	#ifdef VBOX_STRICT
2848	/** @todo NSTVMX: Remove this later. */
2849	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_IA32_SYSENTER_CS);
2850	Assert((fMsrpm & (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR)) == (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR));
2851
2852	fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K8_GS_BASE);
2853	Assert((fMsrpm & (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR)) == (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR));
2854	#endif
2855
2856	/*
2857	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
2858	* associated with then. We never need to intercept access (writes need to be
2859	* executed without causing a VM-exit, reads will #GP fault anyway).
2860	*
2861	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
2862	* read/write them. We swap the the guest/host MSR value using the
2863	* auto-load/store MSR area.
2864	*/
2865	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2866	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2867	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
2868	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2869	if (pVM->cpum.ro.GuestFeatures.fIbrs)
2870	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2871
2872	/*
2873	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
2874	*/
2875
2876	#if HC_ARCH_BITS == 64
2877	/*
2878	* Allow full read/write access for the following MSRs (mandatory for VT-x)
2879	* required for 64-bit guests.
2880	*/
2881	if (pVM->hm.s.fAllow64BitGuests)
2882	{
2883	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2884	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K6_STAR, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2885	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2886	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2887
2888	# ifdef VBOX_STRICT
2889	fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K8_GS_BASE);
2890	Assert((fMsrpm & (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR)) == (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR));
2891	# endif
2892	}
2893	#endif
2894	}
2895
2896
2897	/**
2898	* Sets up pin-based VM-execution controls in the VMCS.
2899	*
2900	* @returns VBox status code.
2901	* @param pVCpu The cross context virtual CPU structure.
2902	* @param pVmcsInfo The VMCS info. object.
2903	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2904	*/
2905	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2906	{
2907	PVM pVM = pVCpu->CTX_SUFF(pVM);
2908	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
2909	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2910
2911	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2912	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2913
2914	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2915	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2916
2917	/* Enable the VMX-preemption timer. */
2918	if (pVM->hm.s.vmx.fUsePreemptTimer)
2919	{
2920	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2921	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
2922	}
2923
2924	#if 0
2925	/* Enable posted-interrupt processing. */
2926	if (pVM->hm.s.fPostedIntrs)
2927	{
2928	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2929	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2930	fVal \|= VMX_PIN_CTL_POSTED_INT;
2931	}
2932	#endif
2933
2934	if ((fVal & fZap) != fVal)
2935	{
2936	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2937	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
2938	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2939	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2940	}
2941
2942	/* Commit it to the VMCS and update our cache. */
2943	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2944	AssertRCReturn(rc, rc);
2945	pVmcsInfo->u32PinCtls = fVal;
2946
2947	return VINF_SUCCESS;
2948	}
2949
2950
2951	/**
2952	* Sets up secondary processor-based VM-execution controls in the VMCS.
2953	*
2954	* @returns VBox status code.
2955	* @param pVCpu The cross context virtual CPU structure.
2956	* @param pVmcsInfo The VMCS info. object.
2957	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2958	*/
2959	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2960	{
2961	PVM pVM = pVCpu->CTX_SUFF(pVM);
2962	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
2963	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2964
2965	/* WBINVD causes a VM-exit. */
2966	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2967	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
2968
2969	/* Enable EPT (aka nested-paging). */
2970	if (pVM->hm.s.fNestedPaging)
2971	fVal \|= VMX_PROC_CTLS2_EPT;
2972
2973	/* Enable the INVPCID instruction if supported by the hardware and we expose
2974	it to the guest. Without this, guest executing INVPCID would cause a #UD. */
2975	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2976	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
2977	fVal \|= VMX_PROC_CTLS2_INVPCID;
2978
2979	/* Enable VPID. */
2980	if (pVM->hm.s.vmx.fVpid)
2981	fVal \|= VMX_PROC_CTLS2_VPID;
2982
2983	/* Enable unrestricted guest execution. */
2984	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2985	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
2986
2987	#if 0
2988	if (pVM->hm.s.fVirtApicRegs)
2989	{
2990	/* Enable APIC-register virtualization. */
2991	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
2992	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
2993
2994	/* Enable virtual-interrupt delivery. */
2995	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
2996	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
2997	}
2998	#endif
2999
3000	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
3001	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3002	* done dynamically. */
3003	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3004	{
3005	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3006	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3007	AssertRCReturn(rc, rc);
3008	}
3009
3010	/* Enable the RDTSCP instruction if supported by the hardware and we expose
3011	it to the guest. Without this, guest executing RDTSCP would cause a #UD. */
3012	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
3013	&& pVM->cpum.ro.GuestFeatures.fRdTscP)
3014	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3015
3016	/* Enable Pause-Loop exiting. */
3017	if ( pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT
3018	&& pVM->hm.s.vmx.cPleGapTicks
3019	&& pVM->hm.s.vmx.cPleWindowTicks)
3020	{
3021	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3022
3023	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3024	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3025	AssertRCReturn(rc, rc);
3026	}
3027
3028	if ((fVal & fZap) != fVal)
3029	{
3030	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3031	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3032	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3033	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3034	}
3035
3036	/* Commit it to the VMCS and update our cache. */
3037	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3038	AssertRCReturn(rc, rc);
3039	pVmcsInfo->u32ProcCtls2 = fVal;
3040
3041	return VINF_SUCCESS;
3042	}
3043
3044
3045	/**
3046	* Sets up processor-based VM-execution controls in the VMCS.
3047	*
3048	* @returns VBox status code.
3049	* @param pVCpu The cross context virtual CPU structure.
3050	* @param pVmcsInfo The VMCS info. object.
3051	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3052	*/
3053	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3054	{
3055	PVM pVM = pVCpu->CTX_SUFF(pVM);
3056
3057	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3058	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3059
3060	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3061	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3062	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3063	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3064	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3065	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3066	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3067
3068	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3069	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3070	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3071	{
3072	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3073	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3074	}
3075
3076	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3077	if (!pVM->hm.s.fNestedPaging)
3078	{
3079	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3080	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3081	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3082	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3083	}
3084
3085	/* Use TPR shadowing if supported by the CPU. */
3086	if ( PDMHasApic(pVM)
3087	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3088	{
3089	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3090	/* CR8 writes cause a VM-exit based on TPR threshold. */
3091	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3092	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3093	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVmcsInfo);
3094	AssertRCReturn(rc, rc);
3095	}
3096	else
3097	{
3098	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3099	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3100	if (pVM->hm.s.fAllow64BitGuests)
3101	{
3102	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3103	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3104	}
3105	}
3106
3107	/* Use MSR-bitmaps if supported by the CPU. */
3108	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3109	{
3110	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3111	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
3112	AssertRCReturn(rc, rc);
3113	}
3114
3115	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3116	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3117	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3118
3119	if ((fVal & fZap) != fVal)
3120	{
3121	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3122	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3123	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3124	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3125	}
3126
3127	/* Commit it to the VMCS and update our cache. */
3128	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3129	AssertRCReturn(rc, rc);
3130	pVmcsInfo->u32ProcCtls = fVal;
3131
3132	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3133	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo, false /* fIsNstGstVmcs */);
3134
3135	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3136	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3137	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3138
3139	/* Sanity check, should not really happen. */
3140	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3141	{ /* likely */ }
3142	else
3143	{
3144	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3145	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3146	}
3147
3148	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3149	return VINF_SUCCESS;
3150	}
3151
3152
3153	/**
3154	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3155	* Processor-based VM-execution) control fields in the VMCS.
3156	*
3157	* @returns VBox status code.
3158	* @param pVCpu The cross context virtual CPU structure.
3159	* @param pVmcsInfo The VMCS info. object.
3160	*/
3161	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3162	{
3163	/* Set the auto-load/store MSR area addresses in the VMCS. */
3164	int rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
3165	if (RT_SUCCESS(rc))
3166	{
3167	/* Set the VMCS link pointer in the VMCS. */
3168	rc = hmR0VmxSetupVmcsLinkPtr(pVmcsInfo);
3169	if (RT_SUCCESS(rc))
3170	{
3171	/* Set the CR0/CR4 guest/host mask. */
3172	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3173	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3174	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3175	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3176	if (RT_SUCCESS(rc))
3177	{
3178	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3179	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3180	return VINF_SUCCESS;
3181	}
3182	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3183	}
3184	else
3185	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3186	}
3187	else
3188	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3189	return rc;
3190	}
3191
3192
3193	/**
3194	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3195	*
3196	* We shall setup those exception intercepts that don't change during the
3197	* lifetime of the VM here. The rest are done dynamically while loading the
3198	* guest state.
3199	*
3200	* @returns VBox status code.
3201	* @param pVCpu The cross context virtual CPU structure.
3202	* @param pVmcsInfo The VMCS info. object.
3203	*/
3204	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3205	{
3206	/*
3207	* The following exceptions are always intercepted:
3208	*
3209	* #AC - To prevent the guest from hanging the CPU.
3210	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3211	* recursive #DBs can cause a CPU hang.
3212	* #PF - To sync our shadow page tables when nested-paging is not used.
3213	*/
3214	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3215	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3216	\| RT_BIT(X86_XCPT_DB)
3217	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3218
3219	/* Commit it to the VMCS. */
3220	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3221	AssertRCReturn(rc, rc);
3222
3223	/* Update our cache of the exception bitmap. */
3224	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3225	return VINF_SUCCESS;
3226	}
3227
3228
3229	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3230	/**
3231	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3232	*
3233	* @returns VBox status code.
3234	* @param pVCpu The cross context virtual CPU structure.
3235	* @param pVmcsInfo The VMCS info. object.
3236	*/
3237	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3238	{
3239	PVM pVM = pVCpu->CTX_SUFF(pVM);
3240	int rc = hmR0VmxSetupVmcsLinkPtr(pVmcsInfo);
3241	if (RT_SUCCESS(rc))
3242	{
3243	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
3244	if (RT_SUCCESS(rc))
3245	{
3246	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3247	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
3248	if (RT_SUCCESS(rc))
3249	{
3250	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3251	rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3252	if (RT_SUCCESS(rc))
3253	return VINF_SUCCESS;
3254
3255	LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3256	}
3257	else
3258	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3259	}
3260	else
3261	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3262	}
3263	else
3264	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3265
3266	return rc;
3267	}
3268	#endif
3269
3270
3271	/**
3272	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3273	* VMX.
3274	*
3275	* @returns VBox status code.
3276	* @param pVCpu The cross context virtual CPU structure.
3277	* @param pVmcsInfo The VMCS info. object.
3278	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3279	*/
3280	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3281	{
3282	Assert(pVmcsInfo);
3283	Assert(pVmcsInfo->pvVmcs);
3284	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3285
3286	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3287	PVM pVM = pVCpu->CTX_SUFF(pVM);
3288	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3289	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3290
3291	LogFlowFunc(("\n"));
3292
3293	/*
3294	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3295	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3296	*/
3297	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3298	if (RT_SUCCESS(rc))
3299	{
3300	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3301	if (RT_SUCCESS(rc))
3302	{
3303	if (!fIsNstGstVmcs)
3304	{
3305	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3306	if (RT_SUCCESS(rc))
3307	{
3308	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
3309	if (RT_SUCCESS(rc))
3310	{
3311	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
3312	if (RT_SUCCESS(rc))
3313	{
3314	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
3315	if (RT_SUCCESS(rc))
3316	{ /* likely */ }
3317	else
3318	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
3319	}
3320	else
3321	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
3322	}
3323	else
3324	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
3325	}
3326	else
3327	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
3328	}
3329	else
3330	{
3331	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3332	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
3333	if (RT_SUCCESS(rc))
3334	{ /* likely */ }
3335	else
3336	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
3337	#else
3338	AssertFailed();
3339	#endif
3340	}
3341	}
3342	else
3343	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
3344	}
3345	else
3346	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
3347
3348	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
3349	if (RT_SUCCESS(rc))
3350	{
3351	rc = hmR0VmxClearVmcs(pVmcsInfo);
3352	if (RT_SUCCESS(rc))
3353	{ /* likely */ }
3354	else
3355	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
3356	}
3357
3358	/*
3359	* Update the last-error record both for failures and success, so we
3360	* can propagate the status code back to ring-3 for diagnostics.
3361	*/
3362	hmR0VmxUpdateErrorRecord(pVCpu, rc);
3363	NOREF(pszVmcs);
3364	return rc;
3365	}
3366
3367
3368	/**
3369	* Does global VT-x initialization (called during module initialization).
3370	*
3371	* @returns VBox status code.
3372	*/
3373	VMMR0DECL(int) VMXR0GlobalInit(void)
3374	{
3375	#ifdef HMVMX_USE_FUNCTION_TABLE
3376	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
3377	# ifdef VBOX_STRICT
3378	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
3379	Assert(g_apfnVMExitHandlers[i]);
3380	# endif
3381	#endif
3382	return VINF_SUCCESS;
3383	}
3384
3385
3386	/**
3387	* Does global VT-x termination (called during module termination).
3388	*/
3389	VMMR0DECL(void) VMXR0GlobalTerm()
3390	{
3391	/* Nothing to do currently. */
3392	}
3393
3394
3395	/**
3396	* Sets up and activates VT-x on the current CPU.
3397	*
3398	* @returns VBox status code.
3399	* @param pHostCpu The HM physical-CPU structure.
3400	* @param pVM The cross context VM structure. Can be
3401	* NULL after a host resume operation.
3402	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
3403	* fEnabledByHost is @c true).
3404	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
3405	* @a fEnabledByHost is @c true).
3406	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
3407	* enable VT-x on the host.
3408	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
3409	*/
3410	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
3411	PCSUPHWVIRTMSRS pHwvirtMsrs)
3412	{
3413	Assert(pHostCpu);
3414	Assert(pHwvirtMsrs);
3415	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3416
3417	/* Enable VT-x if it's not already enabled by the host. */
3418	if (!fEnabledByHost)
3419	{
3420	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
3421	if (RT_FAILURE(rc))
3422	return rc;
3423	}
3424
3425	/*
3426	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
3427	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
3428	* invalidated when flushing by VPID.
3429	*/
3430	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3431	{
3432	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
3433	pHostCpu->fFlushAsidBeforeUse = false;
3434	}
3435	else
3436	pHostCpu->fFlushAsidBeforeUse = true;
3437
3438	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
3439	++pHostCpu->cTlbFlushes;
3440
3441	return VINF_SUCCESS;
3442	}
3443
3444
3445	/**
3446	* Deactivates VT-x on the current CPU.
3447	*
3448	* @returns VBox status code.
3449	* @param pvCpuPage Pointer to the VMXON region.
3450	* @param HCPhysCpuPage Physical address of the VMXON region.
3451	*
3452	* @remarks This function should never be called when SUPR0EnableVTx() or
3453	* similar was used to enable VT-x on the host.
3454	*/
3455	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
3456	{
3457	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
3458
3459	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3460	return hmR0VmxLeaveRootMode();
3461	}
3462
3463
3464	/**
3465	* Does per-VM VT-x initialization.
3466	*
3467	* @returns VBox status code.
3468	* @param pVM The cross context VM structure.
3469	*/
3470	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
3471	{
3472	LogFlowFunc(("pVM=%p\n", pVM));
3473
3474	int rc = hmR0VmxStructsAlloc(pVM);
3475	if (RT_FAILURE(rc))
3476	{
3477	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
3478	return rc;
3479	}
3480
3481	return VINF_SUCCESS;
3482	}
3483
3484
3485	/**
3486	* Does per-VM VT-x termination.
3487	*
3488	* @returns VBox status code.
3489	* @param pVM The cross context VM structure.
3490	*/
3491	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
3492	{
3493	LogFlowFunc(("pVM=%p\n", pVM));
3494
3495	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3496	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
3497	{
3498	Assert(pVM->hm.s.vmx.pvScratch);
3499	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
3500	}
3501	#endif
3502	hmR0VmxStructsFree(pVM);
3503	return VINF_SUCCESS;
3504	}
3505
3506
3507	/**
3508	* Sets up the VM for execution using hardware-assisted VMX.
3509	* This function is only called once per-VM during initialization.
3510	*
3511	* @returns VBox status code.
3512	* @param pVM The cross context VM structure.
3513	*/
3514	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
3515	{
3516	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
3517	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3518
3519	LogFlowFunc(("pVM=%p\n", pVM));
3520
3521	/*
3522	* At least verify if VMX is enabled, since we can't check if we're in
3523	* VMX root mode or not without causing a #GP.
3524	*/
3525	RTCCUINTREG const uHostCR4 = ASMGetCR4();
3526	if (RT_LIKELY(uHostCR4 & X86_CR4_VMXE))
3527	{ /* likely */ }
3528	else
3529	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
3530
3531	/*
3532	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
3533	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
3534	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
3535	*/
3536	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3537	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
3538	\|\| !pVM->hm.s.vmx.pRealModeTSS))
3539	{
3540	LogRelFunc(("Invalid real-on-v86 state.\n"));
3541	return VERR_INTERNAL_ERROR;
3542	}
3543
3544	/* Initialize these always, see hmR3InitFinalizeR0().*/
3545	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
3546	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
3547
3548	/* Setup the tagged-TLB flush handlers. */
3549	int rc = hmR0VmxSetupTaggedTlb(pVM);
3550	if (RT_FAILURE(rc))
3551	{
3552	LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
3553	return rc;
3554	}
3555
3556	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
3557	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
3558	#if HC_ARCH_BITS == 64
3559	if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
3560	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
3561	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
3562	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
3563	#endif
3564
3565	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3566	{
3567	PVMCPU pVCpu = &pVM->aCpus[idCpu];
3568	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
3569
3570	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
3571	if (RT_SUCCESS(rc))
3572	{
3573	#if HC_ARCH_BITS == 32
3574	hmR0VmxInitVmcsReadCache(pVCpu);
3575	#endif
3576	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3577	if (pVM->cpum.ro.GuestFeatures.fVmx)
3578	{
3579	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
3580	if (RT_SUCCESS(rc))
3581	{ /* likely */ }
3582	else
3583	{
3584	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
3585	return rc;
3586	}
3587	}
3588	#endif
3589	}
3590	else
3591	{
3592	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
3593	return rc;
3594	}
3595	}
3596
3597	return VINF_SUCCESS;
3598	}
3599
3600
3601	#if HC_ARCH_BITS == 32
3602	# ifdef VBOX_ENABLE_64_BITS_GUESTS
3603	/**
3604	* Check if guest state allows safe use of 32-bit switcher again.
3605	*
3606	* Segment bases and protected mode structures must be 32-bit addressable
3607	* because the 32-bit switcher will ignore high dword when writing these VMCS
3608	* fields. See @bugref{8432} for details.
3609	*
3610	* @returns true if safe, false if must continue to use the 64-bit switcher.
3611	* @param pCtx Pointer to the guest-CPU context.
3612	*
3613	* @remarks No-long-jump zone!!!
3614	*/
3615	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
3616	{
3617	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
3618	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
3619	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3620	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3621	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
3622	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3623	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
3624	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
3625	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3626	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3627
3628	/* All good, bases are 32-bit. */
3629	return true;
3630	}
3631	# endif
3632
3633	/**
3634	* Executes the specified handler in 64-bit mode.
3635	*
3636	* @returns VBox status code (no informational status codes).
3637	* @param pVCpu The cross context virtual CPU structure.
3638	* @param enmOp The operation to perform.
3639	* @param cParams Number of parameters.
3640	* @param paParam Array of 32-bit parameters.
3641	*/
3642	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
3643	{
3644	PVM pVM = pVCpu->CTX_SUFF(pVM);
3645	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
3646	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
3647	Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
3648	Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
3649
3650	#ifdef VBOX_STRICT
3651	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
3652	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
3653
3654	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
3655	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
3656	#endif
3657
3658	/* Disable interrupts. */
3659	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
3660
3661	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
3662	RTCPUID idHostCpu = RTMpCpuId();
3663	CPUMR0SetLApic(pVCpu, idHostCpu);
3664	#endif
3665
3666	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
3667
3668	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3669	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3670
3671	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
3672	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3673	hmR0VmxClearVmcs(pVmcsInfo);
3674
3675	/* Leave VMX root mode and disable VMX. */
3676	VMXDisable();
3677	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3678
3679	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
3680	CPUMSetHyperEIP(pVCpu, enmOp);
3681	for (int i = (int)cParams - 1; i >= 0; i--)
3682	CPUMPushHyper(pVCpu, paParam[i]);
3683
3684	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
3685
3686	/* Call the switcher. */
3687	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
3688	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
3689
3690	/* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
3691	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
3692
3693	/* Re-enter VMX root mode. */
3694	int rc2 = VMXEnable(HCPhysCpuPage);
3695	if (RT_FAILURE(rc2))
3696	{
3697	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3698	ASMSetFlags(fOldEFlags);
3699	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
3700	return rc2;
3701	}
3702
3703	/* Restore the VMCS as the current VMCS. */
3704	rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
3705	AssertRC(rc2);
3706	Assert(!(ASMGetFlags() & X86_EFL_IF));
3707	ASMSetFlags(fOldEFlags);
3708	return rc;
3709	}
3710
3711
3712	/**
3713	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
3714	* supporting 64-bit guests.
3715	*
3716	* @returns VBox status code.
3717	* @param fResume Whether to VMLAUNCH or VMRESUME.
3718	* @param pCtx Pointer to the guest-CPU context.
3719	* @param pCache Pointer to the VMCS batch cache.
3720	* @param pVM The cross context VM structure.
3721	* @param pVCpu The cross context virtual CPU structure.
3722	*/
3723	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
3724	{
3725	NOREF(fResume);
3726
3727	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3728	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3729
3730	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3731	pCache->uPos = 1;
3732	pCache->interPD = PGMGetInterPaeCR3(pVM);
3733	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
3734	#endif
3735
3736	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3737	pCache->TestIn.HCPhysCpuPage = 0;
3738	pCache->TestIn.HCPhysVmcs = 0;
3739	pCache->TestIn.pCache = 0;
3740	pCache->TestOut.HCPhysVmcs = 0;
3741	pCache->TestOut.pCache = 0;
3742	pCache->TestOut.pCtx = 0;
3743	pCache->TestOut.eflags = 0;
3744	#else
3745	NOREF(pCache);
3746	#endif
3747
3748	uint32_t aParam[10];
3749	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
3750	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
3751	aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
3752	aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
3753	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
3754	aParam[5] = 0;
3755	aParam[6] = VM_RC_ADDR(pVM, pVM);
3756	aParam[7] = 0;
3757	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
3758	aParam[9] = 0;
3759
3760	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3761	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
3762	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
3763	#endif
3764	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
3765
3766	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3767	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
3768	Assert(pCtx->dr[4] == 10);
3769	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
3770	#endif
3771
3772	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3773	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
3774	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3775	pVCpu->hm.s.vmx.HCPhysVmcs));
3776	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3777	pCache->TestOut.HCPhysVmcs));
3778	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
3779	pCache->TestOut.pCache));
3780	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
3781	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
3782	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
3783	pCache->TestOut.pCtx));
3784	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
3785	#endif
3786	NOREF(pCtx);
3787	return rc;
3788	}
3789	#endif
3790
3791
3792	/**
3793	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
3794	* the VMCS.
3795	*
3796	* @returns VBox status code.
3797	*/
3798	static int hmR0VmxExportHostControlRegs(void)
3799	{
3800	RTCCUINTREG uReg = ASMGetCR0();
3801	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
3802	AssertRCReturn(rc, rc);
3803
3804	uReg = ASMGetCR3();
3805	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
3806	AssertRCReturn(rc, rc);
3807
3808	uReg = ASMGetCR4();
3809	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
3810	AssertRCReturn(rc, rc);
3811	return rc;
3812	}
3813
3814
3815	/**
3816	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
3817	* the host-state area in the VMCS.
3818	*
3819	* @returns VBox status code.
3820	* @param pVCpu The cross context virtual CPU structure.
3821	*/
3822	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
3823	{
3824	#if HC_ARCH_BITS == 64
3825	/**
3826	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
3827	* requirements. See hmR0VmxExportHostSegmentRegs().
3828	*/
3829	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
3830	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
3831	{ \
3832	bool fValidSelector = true; \
3833	if ((selValue) & X86_SEL_LDT) \
3834	{ \
3835	uint32_t uAttr = ASMGetSegAttr((selValue)); \
3836	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
3837	} \
3838	if (fValidSelector) \
3839	{ \
3840	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
3841	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
3842	} \
3843	(selValue) = 0; \
3844	}
3845
3846	/*
3847	* If we've executed guest code using hardware-assisted VMX, the host-state bits
3848	* will be messed up. We should -not- save the messed up state without restoring
3849	* the original host-state, see @bugref{7240}.
3850	*
3851	* This apparently can happen (most likely the FPU changes), deal with it rather than
3852	* asserting. Was observed booting Solaris 10u10 32-bit guest.
3853	*/
3854	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
3855	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
3856	{
3857	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
3858	pVCpu->idCpu));
3859	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
3860	}
3861	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
3862	#else
3863	RT_NOREF(pVCpu);
3864	#endif
3865
3866	/*
3867	* Host DS, ES, FS and GS segment registers.
3868	*/
3869	#if HC_ARCH_BITS == 64
3870	RTSEL uSelDS = ASMGetDS();
3871	RTSEL uSelES = ASMGetES();
3872	RTSEL uSelFS = ASMGetFS();
3873	RTSEL uSelGS = ASMGetGS();
3874	#else
3875	RTSEL uSelDS = 0;
3876	RTSEL uSelES = 0;
3877	RTSEL uSelFS = 0;
3878	RTSEL uSelGS = 0;
3879	#endif
3880
3881	/*
3882	* Host CS and SS segment registers.
3883	*/
3884	RTSEL uSelCS = ASMGetCS();
3885	RTSEL uSelSS = ASMGetSS();
3886
3887	/*
3888	* Host TR segment register.
3889	*/
3890	RTSEL uSelTR = ASMGetTR();
3891
3892	#if HC_ARCH_BITS == 64
3893	/*
3894	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
3895	* gain VM-entry and restore them before we get preempted.
3896	*
3897	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
3898	*/
3899	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3900	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3901	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3902	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3903	# undef VMXLOCAL_ADJUST_HOST_SEG
3904	#endif
3905
3906	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
3907	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
3908	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
3909	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
3910	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
3911	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
3912	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
3913	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
3914	Assert(uSelCS);
3915	Assert(uSelTR);
3916
3917	/* Write these host selector fields into the host-state area in the VMCS. */
3918	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3919	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3920	#if HC_ARCH_BITS == 64
3921	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3922	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3923	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3924	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3925	#else
3926	NOREF(uSelDS);
3927	NOREF(uSelES);
3928	NOREF(uSelFS);
3929	NOREF(uSelGS);
3930	#endif
3931	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3932	AssertRCReturn(rc, rc);
3933
3934	/*
3935	* Host GDTR and IDTR.
3936	*/
3937	RTGDTR Gdtr;
3938	RTIDTR Idtr;
3939	RT_ZERO(Gdtr);
3940	RT_ZERO(Idtr);
3941	ASMGetGDTR(&Gdtr);
3942	ASMGetIDTR(&Idtr);
3943	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3944	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3945	AssertRCReturn(rc, rc);
3946
3947	#if HC_ARCH_BITS == 64
3948	/*
3949	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
3950	* them to the maximum limit (0xffff) on every VM-exit.
3951	*/
3952	if (Gdtr.cbGdt != 0xffff)
3953	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
3954
3955	/*
3956	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
3957	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
3958	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
3959	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
3960	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
3961	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
3962	* at 0xffff on hosts where we are sure it won't cause trouble.
3963	*/
3964	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
3965	if (Idtr.cbIdt < 0x0fff)
3966	# else
3967	if (Idtr.cbIdt != 0xffff)
3968	# endif
3969	{
3970	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
3971	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3972	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3973	}
3974	#endif
3975
3976	/*
3977	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3978	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3979	* RPL should be too in most cases.
3980	*/
3981	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3982	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3983
3984	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3985	#if HC_ARCH_BITS == 64
3986	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
3987
3988	/*
3989	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3990	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3991	* restoration if the host has something else. Task switching is not supported in 64-bit
3992	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3993	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3994	*
3995	* [1] See Intel spec. 3.5 "System Descriptor Types".
3996	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3997	*/
3998	PVM pVM = pVCpu->CTX_SUFF(pVM);
3999	Assert(pDesc->System.u4Type == 11);
4000	if ( pDesc->System.u16LimitLow != 0x67
4001	\|\| pDesc->System.u4LimitHigh)
4002	{
4003	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4004	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4005	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4006	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4007	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4008	}
4009
4010	/*
4011	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4012	*/
4013	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4014	{
4015	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4016	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4017	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4018	{
4019	/* The GDT is read-only but the writable GDT is available. */
4020	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4021	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4022	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4023	AssertRCReturn(rc, rc);
4024	}
4025	}
4026	#else
4027	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4028	#endif
4029	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4030	AssertRCReturn(rc, rc);
4031
4032	/*
4033	* Host FS base and GS base.
4034	*/
4035	#if HC_ARCH_BITS == 64
4036	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4037	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4038	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4039	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4040	AssertRCReturn(rc, rc);
4041
4042	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4043	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4044	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4045	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4046	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4047	#endif
4048	return VINF_SUCCESS;
4049	}
4050
4051
4052	/**
4053	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4054	* host-state area of the VMCS.
4055	*
4056	* These MSRs will be automatically restored on the host after every successful
4057	* VM-exit.
4058	*
4059	* @returns VBox status code.
4060	* @param pVCpu The cross context virtual CPU structure.
4061	*
4062	* @remarks No-long-jump zone!!!
4063	*/
4064	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4065	{
4066	AssertPtr(pVCpu);
4067
4068	/*
4069	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4070	* rather than swapping them on every VM-entry.
4071	*/
4072	hmR0VmxLazySaveHostMsrs(pVCpu);
4073
4074	/*
4075	* Host Sysenter MSRs.
4076	*/
4077	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4078	#if HC_ARCH_BITS == 32
4079	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4080	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4081	#else
4082	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4083	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4084	#endif
4085	AssertRCReturn(rc, rc);
4086
4087	/*
4088	* Host EFER MSR.
4089	*
4090	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4091	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4092	*/
4093	PVM pVM = pVCpu->CTX_SUFF(pVM);
4094	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4095	{
4096	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4097	AssertRCReturn(rc, rc);
4098	}
4099
4100	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4101	* hmR0VmxExportGuestEntryExitCtls(). */
4102
4103	return VINF_SUCCESS;
4104	}
4105
4106
4107	/**
4108	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4109	*
4110	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4111	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4112	*
4113	* @returns true if we need to load guest EFER, false otherwise.
4114	* @param pVCpu The cross context virtual CPU structure.
4115	*
4116	* @remarks Requires EFER, CR4.
4117	* @remarks No-long-jump zone!!!
4118	*/
4119	static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu)
4120	{
4121	#ifdef HMVMX_ALWAYS_SWAP_EFER
4122	RT_NOREF(pVCpu);
4123	return true;
4124	#else
4125	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4126	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4127	/* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4128	if (CPUMIsGuestInLongModeEx(pCtx))
4129	return false;
4130	#endif
4131
4132	PVM pVM = pVCpu->CTX_SUFF(pVM);
4133	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4134	uint64_t const u64GuestEfer = pCtx->msrEFER;
4135
4136	/*
4137	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4138	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4139	*/
4140	if ( CPUMIsGuestInLongModeEx(pCtx)
4141	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4142	return true;
4143
4144	/*
4145	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4146	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4147	*
4148	* See Intel spec. 4.5 "IA-32e Paging".
4149	* See Intel spec. 4.1.1 "Three Paging Modes".
4150	*
4151	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4152	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4153	*/
4154	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4155	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4156	if ( (pCtx->cr4 & X86_CR4_PAE)
4157	&& (pCtx->cr0 & X86_CR0_PG)
4158	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4159	{
4160	/* Assert that host is NX capable. */
4161	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4162	return true;
4163	}
4164
4165	return false;
4166	#endif
4167	}
4168
4169	/**
4170	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4171	* VMCS.
4172	*
4173	* This is typically required when the guest changes paging mode.
4174	*
4175	* @returns VBox status code.
4176	* @param pVCpu The cross context virtual CPU structure.
4177	* @param pVmxTransient The VMX-transient structure.
4178	*
4179	* @remarks Requires EFER.
4180	* @remarks No-long-jump zone!!!
4181	*/
4182	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4183	{
4184	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4185	{
4186	PVM pVM = pVCpu->CTX_SUFF(pVM);
4187	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4188
4189	/*
4190	* VM-entry controls.
4191	*/
4192	{
4193	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4194	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4195
4196	/*
4197	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4198	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4199	*
4200	* For nested-guests, this is a mandatory VM-entry control. It's also
4201	* required because we do not want to leak host bits to the nested-guest.
4202	*/
4203	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4204
4205	/*
4206	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4207	*
4208	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4209	* required to get the nested-guest working with hardware-assisted VMX execution.
4210	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested-hypervisor
4211	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4212	* here rather than while merging the guest VMCS controls.
4213	*/
4214	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4215	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4216	else
4217	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4218
4219	/*
4220	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4221	*
4222	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4223	* regardless of whether the nested-guest VMCS specifies it because we are free to
4224	* load whatever MSRs we require and we do not need to modify the guest visible copy
4225	* of the VM-entry MSR load area.
4226	*/
4227	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4228	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4229	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4230	else
4231	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4232
4233	/*
4234	* The following should -not- be set (since we're not in SMM mode):
4235	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4236	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4237	*/
4238
4239	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4240	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4241
4242	if ((fVal & fZap) == fVal)
4243	{ /* likely */ }
4244	else
4245	{
4246	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4247	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4248	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4249	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4250	}
4251
4252	/* Commit it to the VMCS. */
4253	if (pVmcsInfo->u32EntryCtls != fVal)
4254	{
4255	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4256	AssertRCReturn(rc, rc);
4257	pVmcsInfo->u32EntryCtls = fVal;
4258	}
4259	}
4260
4261	/*
4262	* VM-exit controls.
4263	*/
4264	{
4265	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4266	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4267
4268	/*
4269	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4270	* supported the 1-setting of this bit.
4271	*
4272	* For nested-guests, we set the "save debug controls" as the converse
4273	* "load debug controls" is mandatory for nested-guests anyway.
4274	*/
4275	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4276
4277	/*
4278	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4279	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4280	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4281	* hmR0VmxExportHostMsrs().
4282	*
4283	* For nested-guests, we always set this bit as we do not support 32-bit
4284	* hosts.
4285	*/
4286	#if HC_ARCH_BITS == 64
4287	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4288	#else
4289	Assert(!pVmxTransient->fIsNestedGuest);
4290	Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
4291	\|\| pVmcsInfo->pfnStartVM == VMXR0StartVM32);
4292	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
4293	if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
4294	{
4295	/* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
4296	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4297	}
4298	else
4299	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
4300	#endif
4301
4302	/*
4303	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4304	*
4305	* For nested-guests, we should use the "save IA32_EFER" control if we also
4306	* used the "load IA32_EFER" control while exporting VM-entry controls.
4307	*/
4308	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4309	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4310	{
4311	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4312	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4313	}
4314
4315	/*
4316	* Enable saving of the VMX-preemption timer value on VM-exit.
4317	* For nested-guests, currently not exposed/used.
4318	*/
4319	if ( pVM->hm.s.vmx.fUsePreemptTimer
4320	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4321	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4322
4323	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4324	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4325
4326	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4327	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4328	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4329
4330	if ((fVal & fZap) == fVal)
4331	{ /* likely */ }
4332	else
4333	{
4334	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4335	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4336	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4337	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4338	}
4339
4340	/* Commit it to the VMCS. */
4341	if (pVmcsInfo->u32ExitCtls != fVal)
4342	{
4343	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4344	AssertRCReturn(rc, rc);
4345	pVmcsInfo->u32ExitCtls = fVal;
4346	}
4347	}
4348
4349	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4350	}
4351	return VINF_SUCCESS;
4352	}
4353
4354
4355	/**
4356	* Sets the TPR threshold in the VMCS.
4357	*
4358	* @returns VBox status code.
4359	* @param pVCpu The cross context virtual CPU structure.
4360	* @param pVmcsInfo The VMCS info. object.
4361	* @param u32TprThreshold The TPR threshold (task-priority class only).
4362	*/
4363	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4364	{
4365	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4366	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4367	RT_NOREF2(pVCpu, pVmcsInfo);
4368	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4369	}
4370
4371
4372	/**
4373	* Exports the guest APIC TPR state into the VMCS.
4374	*
4375	* @returns VBox status code.
4376	* @param pVCpu The cross context virtual CPU structure.
4377	* @param pVmxTransient The VMX-transient structure.
4378	*
4379	* @remarks No-long-jump zone!!!
4380	*/
4381	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4382	{
4383	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4384	{
4385	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4386
4387	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4388	if (!pVmxTransient->fIsNestedGuest)
4389	{
4390	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4391	&& APICIsEnabled(pVCpu))
4392	{
4393	/*
4394	* Setup TPR shadowing.
4395	*/
4396	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4397	{
4398	bool fPendingIntr = false;
4399	uint8_t u8Tpr = 0;
4400	uint8_t u8PendingIntr = 0;
4401	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4402	AssertRCReturn(rc, rc);
4403
4404	/*
4405	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4406	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4407	* priority of the pending interrupt so we can deliver the interrupt. If there
4408	* are no interrupts pending, set threshold to 0 to not cause any
4409	* TPR-below-threshold VM-exits.
4410	*/
4411	Assert(pVmcsInfo->pbVirtApic);
4412	pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
4413	uint32_t u32TprThreshold = 0;
4414	if (fPendingIntr)
4415	{
4416	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4417	(which is the Task-Priority Class). */
4418	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4419	const uint8_t u8TprPriority = u8Tpr >> 4;
4420	if (u8PendingPriority <= u8TprPriority)
4421	u32TprThreshold = u8PendingPriority;
4422	}
4423
4424	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
4425	AssertRCReturn(rc, rc);
4426	}
4427	}
4428	}
4429	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4430	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4431	}
4432	return VINF_SUCCESS;
4433	}
4434
4435
4436	/**
4437	* Gets the guest interruptibility-state.
4438	*
4439	* @returns Guest's interruptibility-state.
4440	* @param pVCpu The cross context virtual CPU structure.
4441	* @param pVmcsInfo The VMCS info. object.
4442	*
4443	* @remarks No-long-jump zone!!!
4444	*/
4445	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
4446	{
4447	/*
4448	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4449	*/
4450	uint32_t fIntrState = 0;
4451	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4452	{
4453	/* If inhibition is active, RIP and RFLAGS should've been updated
4454	(i.e. read previously from the VMCS or from ring-3). */
4455	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4456	#ifdef VBOX_STRICT
4457	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
4458	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4459	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
4460	#endif
4461	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
4462	{
4463	if (pCtx->eflags.Bits.u1IF)
4464	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4465	else
4466	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
4467	}
4468	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4469	{
4470	/*
4471	* We can clear the inhibit force flag as even if we go back to the recompiler
4472	* without executing guest code in VT-x, the flag's condition to be cleared is
4473	* met and thus the cleared state is correct.
4474	*/
4475	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4476	}
4477	}
4478
4479	/*
4480	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
4481	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
4482	* setting this would block host-NMIs and IRET will not clear the blocking.
4483	*
4484	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
4485	*
4486	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
4487	*/
4488	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
4489	&& CPUMIsGuestNmiBlocking(pVCpu))
4490	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
4491
4492	return fIntrState;
4493	}
4494
4495
4496	/**
4497	* Exports the exception intercepts required for guest execution in the VMCS.
4498	*
4499	* @returns VBox status code.
4500	* @param pVCpu The cross context virtual CPU structure.
4501	* @param pVmxTransient The VMX-transient structure.
4502	*
4503	* @remarks No-long-jump zone!!!
4504	*/
4505	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4506	{
4507	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
4508	{
4509	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
4510	if ( !pVmxTransient->fIsNestedGuest
4511	&& pVCpu->hm.s.fGIMTrapXcptUD)
4512	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
4513	else
4514	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
4515
4516	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
4517	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
4518	}
4519	return VINF_SUCCESS;
4520	}
4521
4522
4523	/**
4524	* Exports the guest's RIP into the guest-state area in the VMCS.
4525	*
4526	* @returns VBox status code.
4527	* @param pVCpu The cross context virtual CPU structure.
4528	*
4529	* @remarks No-long-jump zone!!!
4530	*/
4531	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
4532	{
4533	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4534	{
4535	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
4536
4537	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
4538	AssertRCReturn(rc, rc);
4539
4540	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
4541	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
4542	}
4543	return VINF_SUCCESS;
4544	}
4545
4546
4547	/**
4548	* Exports the guest's RSP into the guest-state area in the VMCS.
4549	*
4550	* @returns VBox status code.
4551	* @param pVCpu The cross context virtual CPU structure.
4552	*
4553	* @remarks No-long-jump zone!!!
4554	*/
4555	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
4556	{
4557	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
4558	{
4559	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
4560
4561	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
4562	AssertRCReturn(rc, rc);
4563
4564	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
4565	}
4566	return VINF_SUCCESS;
4567	}
4568
4569
4570	/**
4571	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
4572	*
4573	* @returns VBox status code.
4574	* @param pVCpu The cross context virtual CPU structure.
4575	* @param pVmxTransient The VMX-transient structure.
4576	*
4577	* @remarks No-long-jump zone!!!
4578	*/
4579	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4580	{
4581	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
4582	{
4583	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
4584
4585	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
4586	Let us assert it as such and use 32-bit VMWRITE. */
4587	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
4588	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
4589	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
4590	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
4591
4592	/*
4593	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
4594	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
4595	* can run the real-mode guest code under Virtual 8086 mode.
4596	*/
4597	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4598	if (pVmcsInfo->RealMode.fRealOnV86Active)
4599	{
4600	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4601	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4602	Assert(!pVmxTransient->fIsNestedGuest);
4603	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
4604	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
4605	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
4606	}
4607
4608	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
4609	AssertRCReturn(rc, rc);
4610
4611	/*
4612	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
4613	*
4614	* We must avoid setting any automatic debug exceptions delivery when single-stepping
4615	* through the hypervisor debugger using EFLAGS.TF.
4616	*/
4617	if ( !pVmxTransient->fIsNestedGuest
4618	&& !pVCpu->hm.s.fSingleInstruction
4619	&& fEFlags.Bits.u1TF)
4620	{
4621	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
4622	* premature trips to ring-3 esp since IEM does not yet handle it. */
4623	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
4624	AssertRCReturn(rc, rc);
4625	}
4626	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
4627	* nested-guest VMCS. */
4628
4629	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
4630	Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
4631	}
4632	return VINF_SUCCESS;
4633	}
4634
4635
4636	/**
4637	* Exports the guest CR0 control register into the guest-state area in the VMCS.
4638	*
4639	* The guest FPU state is always pre-loaded hence we don't need to bother about
4640	* sharing FPU related CR0 bits between the guest and host.
4641	*
4642	* @returns VBox status code.
4643	* @param pVCpu The cross context virtual CPU structure.
4644	* @param pVmxTransient The VMX-transient structure.
4645	*
4646	* @remarks No-long-jump zone!!!
4647	*/
4648	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4649	{
4650	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
4651	{
4652	PVM pVM = pVCpu->CTX_SUFF(pVM);
4653	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4654
4655	/*
4656	* Figure out fixed CR0 bits in VMX operation.
4657	*/
4658	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4659	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4660	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4661	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
4662	else
4663	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
4664
4665	if (!pVmxTransient->fIsNestedGuest)
4666	{
4667	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4668	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4669	uint64_t const u64ShadowCr0 = u64GuestCr0;
4670	Assert(!RT_HI_U32(u64GuestCr0));
4671
4672	/*
4673	* Setup VT-x's view of the guest CR0.
4674	*/
4675	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
4676	if (pVM->hm.s.fNestedPaging)
4677	{
4678	if (CPUMIsGuestPagingEnabled(pVCpu))
4679	{
4680	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
4681	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
4682	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
4683	}
4684	else
4685	{
4686	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
4687	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
4688	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
4689	}
4690
4691	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
4692	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4693	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
4694	}
4695	else
4696	{
4697	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
4698	u64GuestCr0 \|= X86_CR0_WP;
4699	}
4700
4701	/*
4702	* Guest FPU bits.
4703	*
4704	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
4705	* using CR0.TS.
4706	*
4707	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
4708	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
4709	*/
4710	u64GuestCr0 \|= X86_CR0_NE;
4711
4712	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
4713	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
4714
4715	/*
4716	* Update exception intercepts.
4717	*/
4718	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
4719	if (pVmcsInfo->RealMode.fRealOnV86Active)
4720	{
4721	Assert(PDMVmmDevHeapIsEnabled(pVM));
4722	Assert(pVM->hm.s.vmx.pRealModeTSS);
4723	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
4724	}
4725	else
4726	{
4727	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
4728	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
4729	if (fInterceptMF)
4730	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
4731	}
4732
4733	/* Additional intercepts for debugging, define these yourself explicitly. */
4734	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4735	uXcptBitmap \|= 0
4736	\| RT_BIT(X86_XCPT_BP)
4737	\| RT_BIT(X86_XCPT_DE)
4738	\| RT_BIT(X86_XCPT_NM)
4739	\| RT_BIT(X86_XCPT_TS)
4740	\| RT_BIT(X86_XCPT_UD)
4741	\| RT_BIT(X86_XCPT_NP)
4742	\| RT_BIT(X86_XCPT_SS)
4743	\| RT_BIT(X86_XCPT_GP)
4744	\| RT_BIT(X86_XCPT_PF)
4745	\| RT_BIT(X86_XCPT_MF)
4746	;
4747	#elif defined(HMVMX_ALWAYS_TRAP_PF)
4748	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
4749	#endif
4750	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
4751	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
4752	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
4753
4754	/* Apply the fixed CR0 bits and enable caching. */
4755	u64GuestCr0 \|= fSetCr0;
4756	u64GuestCr0 &= fZapCr0;
4757	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4758
4759	/* Commit the CR0 and related fields to the guest VMCS. */
4760	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCr0);
4761	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4762	if (uProcCtls != pVmcsInfo->u32ProcCtls)
4763	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4764	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
4765	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
4766	AssertRCReturn(rc, rc);
4767
4768	/* Update our caches. */
4769	pVmcsInfo->u32ProcCtls = uProcCtls;
4770	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
4771
4772	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4773	}
4774	else
4775	{
4776	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
4777	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4778	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4779	uint64_t const u64ShadowCr0 = pVmcsNstGst->u64Cr0ReadShadow.u;
4780	Assert(!RT_HI_U32(u64GuestCr0));
4781	Assert(u64GuestCr0 & X86_CR0_NE);
4782
4783	/* Apply the fixed CR0 bits and enable caching. */
4784	u64GuestCr0 \|= fSetCr0;
4785	u64GuestCr0 &= fZapCr0;
4786	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4787
4788	/* Commit the CR0 and CR0 read shadow to the nested-guest VMCS. */
4789	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCr0);
4790	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4791	AssertRCReturn(rc, rc);
4792
4793	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4794	}
4795
4796	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
4797	}
4798
4799	return VINF_SUCCESS;
4800	}
4801
4802
4803	/**
4804	* Exports the guest control registers (CR3, CR4) into the guest-state area
4805	* in the VMCS.
4806	*
4807	* @returns VBox strict status code.
4808	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
4809	* without unrestricted guest access and the VMMDev is not presently
4810	* mapped (e.g. EFI32).
4811	*
4812	* @param pVCpu The cross context virtual CPU structure.
4813	* @param pVmxTransient The VMX-transient structure.
4814	*
4815	* @remarks No-long-jump zone!!!
4816	*/
4817	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4818	{
4819	int rc = VINF_SUCCESS;
4820	PVM pVM = pVCpu->CTX_SUFF(pVM);
4821
4822	/*
4823	* Guest CR2.
4824	* It's always loaded in the assembler code. Nothing to do here.
4825	*/
4826
4827	/*
4828	* Guest CR3.
4829	*/
4830	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
4831	{
4832	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
4833
4834	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
4835	if (pVM->hm.s.fNestedPaging)
4836	{
4837	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4838	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
4839
4840	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
4841	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
4842	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
4843	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
4844
4845	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
4846	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
4847	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
4848
4849	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
4850	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
4851	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
4852	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
4853	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
4854	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
4855	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
4856
4857	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
4858	AssertRCReturn(rc, rc);
4859
4860	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4861	if ( pVM->hm.s.vmx.fUnrestrictedGuest
4862	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
4863	{
4864	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
4865	if (CPUMIsGuestInPAEModeEx(pCtx))
4866	{
4867	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
4868	AssertRCReturn(rc, rc);
4869	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
4870	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
4871	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
4872	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
4873	AssertRCReturn(rc, rc);
4874	}
4875
4876	/*
4877	* The guest's view of its CR3 is unblemished with nested paging when the
4878	* guest is using paging or we have unrestricted guest execution to handle
4879	* the guest when it's not using paging.
4880	*/
4881	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
4882	GCPhysGuestCR3 = pCtx->cr3;
4883	}
4884	else
4885	{
4886	/*
4887	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
4888	* thinks it accesses physical memory directly, we use our identity-mapped
4889	* page table to map guest-linear to guest-physical addresses. EPT takes care
4890	* of translating it to host-physical addresses.
4891	*/
4892	RTGCPHYS GCPhys;
4893	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4894
4895	/* We obtain it here every time as the guest could have relocated this PCI region. */
4896	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4897	if (RT_SUCCESS(rc))
4898	{ /* likely */ }
4899	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4900	{
4901	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
4902	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4903	}
4904	else
4905	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4906
4907	GCPhysGuestCR3 = GCPhys;
4908	}
4909
4910	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
4911	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4912	AssertRCReturn(rc, rc);
4913	}
4914	else
4915	{
4916	/* Non-nested paging case, just use the hypervisor's CR3. */
4917	RTHCPHYS const HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4918
4919	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
4920	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4921	AssertRCReturn(rc, rc);
4922	}
4923
4924	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
4925	}
4926
4927	/*
4928	* Guest CR4.
4929	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4930	*/
4931	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
4932	{
4933	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4934	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4935	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
4936
4937	/*
4938	* Figure out fixed CR4 bits in VMX operation.
4939	*/
4940	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
4941	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
4942
4943	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
4944	uint64_t u64GuestCr4 = pCtx->cr4;
4945	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest ? pCtx->cr4 : pVmcsNstGst->u64Cr4ReadShadow.u;
4946	Assert(!RT_HI_U32(u64GuestCr4));
4947
4948	/*
4949	* Setup VT-x's view of the guest CR4.
4950	*
4951	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
4952	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
4953	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4954	*
4955	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4956	*/
4957	if (pVmcsInfo->RealMode.fRealOnV86Active)
4958	{
4959	Assert(pVM->hm.s.vmx.pRealModeTSS);
4960	Assert(PDMVmmDevHeapIsEnabled(pVM));
4961	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
4962	}
4963
4964	if (pVM->hm.s.fNestedPaging)
4965	{
4966	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
4967	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
4968	{
4969	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4970	u64GuestCr4 \|= X86_CR4_PSE;
4971	/* Our identity mapping is a 32-bit page directory. */
4972	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
4973	}
4974	/* else use guest CR4.*/
4975	}
4976	else
4977	{
4978	Assert(!pVmxTransient->fIsNestedGuest);
4979
4980	/*
4981	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4982	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4983	*/
4984	switch (pVCpu->hm.s.enmShadowMode)
4985	{
4986	case PGMMODE_REAL: /* Real-mode. */
4987	case PGMMODE_PROTECTED: /* Protected mode without paging. */
4988	case PGMMODE_32_BIT: /* 32-bit paging. */
4989	{
4990	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
4991	break;
4992	}
4993
4994	case PGMMODE_PAE: /* PAE paging. */
4995	case PGMMODE_PAE_NX: /* PAE paging with NX. */
4996	{
4997	u64GuestCr4 \|= X86_CR4_PAE;
4998	break;
4999	}
5000
5001	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5002	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5003	#ifdef VBOX_ENABLE_64_BITS_GUESTS
5004	break;
5005	#endif
5006	default:
5007	AssertFailed();
5008	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5009	}
5010	}
5011
5012	/* Apply the fixed CR4 bits (mainly CR4.VMXE). */
5013	u64GuestCr4 \|= fSetCr4;
5014	u64GuestCr4 &= fZapCr4;
5015
5016	/* Commit the CR4 and CR4 read shadow to the guest VMCS. */
5017	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR4, u64GuestCr4);
5018	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
5019	AssertRCReturn(rc, rc);
5020
5021	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5022	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5023
5024	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5025
5026	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5027	}
5028	return rc;
5029	}
5030
5031
5032	/**
5033	* Exports the guest debug registers into the guest-state area in the VMCS.
5034	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5035	*
5036	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5037	*
5038	* @returns VBox status code.
5039	* @param pVCpu The cross context virtual CPU structure.
5040	* @param pVmxTransient The VMX-transient structure.
5041	*
5042	* @remarks No-long-jump zone!!!
5043	*/
5044	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5045	{
5046	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5047
5048	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5049	* stepping. */
5050	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5051	if (pVmxTransient->fIsNestedGuest)
5052	{
5053	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5054	AssertRCReturn(rc, rc);
5055	return VINF_SUCCESS;
5056	}
5057
5058	#ifdef VBOX_STRICT
5059	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5060	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5061	{
5062	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5063	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5064	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5065	}
5066	#endif
5067
5068	bool fSteppingDB = false;
5069	bool fInterceptMovDRx = false;
5070	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5071	if (pVCpu->hm.s.fSingleInstruction)
5072	{
5073	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5074	PVM pVM = pVCpu->CTX_SUFF(pVM);
5075	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5076	{
5077	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5078	Assert(fSteppingDB == false);
5079	}
5080	else
5081	{
5082	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5083	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5084	pVCpu->hm.s.fClearTrapFlag = true;
5085	fSteppingDB = true;
5086	}
5087	}
5088
5089	uint32_t u32GuestDr7;
5090	if ( fSteppingDB
5091	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5092	{
5093	/*
5094	* Use the combined guest and host DRx values found in the hypervisor register set
5095	* because the hypervisor debugger has breakpoints active or someone is single stepping
5096	* on the host side without a monitor trap flag.
5097	*
5098	* Note! DBGF expects a clean DR6 state before executing guest code.
5099	*/
5100	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5101	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5102	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
5103	{
5104	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5105	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
5106	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
5107	}
5108	else
5109	#endif
5110	if (!CPUMIsHyperDebugStateActive(pVCpu))
5111	{
5112	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5113	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5114	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5115	}
5116
5117	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5118	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
5119	pVCpu->hm.s.fUsingHyperDR7 = true;
5120	fInterceptMovDRx = true;
5121	}
5122	else
5123	{
5124	/*
5125	* If the guest has enabled debug registers, we need to load them prior to
5126	* executing guest code so they'll trigger at the right time.
5127	*/
5128	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5129	{
5130	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5131	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5132	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
5133	{
5134	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5135	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
5136	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
5137	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5138	}
5139	else
5140	#endif
5141	if (!CPUMIsGuestDebugStateActive(pVCpu))
5142	{
5143	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5144	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5145	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5146	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5147	}
5148	Assert(!fInterceptMovDRx);
5149	}
5150	/*
5151	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5152	* must intercept #DB in order to maintain a correct DR6 guest value, and
5153	* because we need to intercept it to prevent nested #DBs from hanging the
5154	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5155	*/
5156	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5157	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
5158	&& !CPUMIsGuestDebugStateActive(pVCpu))
5159	#else
5160	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5161	#endif
5162	{
5163	fInterceptMovDRx = true;
5164	}
5165
5166	/* Update DR7 with the actual guest value. */
5167	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5168	pVCpu->hm.s.fUsingHyperDR7 = false;
5169	}
5170
5171	if (fInterceptMovDRx)
5172	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5173	else
5174	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5175
5176	/*
5177	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5178	* monitor-trap flag and update our cache.
5179	*/
5180	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5181	{
5182	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5183	AssertRCReturn(rc2, rc2);
5184	pVmcsInfo->u32ProcCtls = uProcCtls;
5185	}
5186
5187	/*
5188	* Update guest DR7.
5189	*/
5190	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
5191	AssertRCReturn(rc, rc);
5192
5193	/*
5194	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
5195	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
5196	*
5197	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5198	*/
5199	if (fSteppingDB)
5200	{
5201	Assert(pVCpu->hm.s.fSingleInstruction);
5202	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
5203
5204	uint32_t fIntrState = 0;
5205	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
5206	AssertRCReturn(rc, rc);
5207
5208	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5209	{
5210	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5211	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5212	AssertRCReturn(rc, rc);
5213	}
5214	}
5215
5216	return VINF_SUCCESS;
5217	}
5218
5219
5220	#ifdef VBOX_STRICT
5221	/**
5222	* Strict function to validate segment registers.
5223	*
5224	* @param pVCpu The cross context virtual CPU structure.
5225	* @param pVmcsInfo The VMCS info. object.
5226	*
5227	* @remarks Will import guest CR0 on strict builds during validation of
5228	* segments.
5229	*/
5230	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
5231	{
5232	/*
5233	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5234	*
5235	* The reason we check for attribute value 0 in this function and not just the unusable bit is
5236	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
5237	* unusable bit and doesn't change the guest-context value.
5238	*/
5239	PVM pVM = pVCpu->CTX_SUFF(pVM);
5240	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5241	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
5242	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
5243	&& ( !CPUMIsGuestInRealModeEx(pCtx)
5244	&& !CPUMIsGuestInV86ModeEx(pCtx)))
5245	{
5246	/* Protected mode checks */
5247	/* CS */
5248	Assert(pCtx->cs.Attr.n.u1Present);
5249	Assert(!(pCtx->cs.Attr.u & 0xf00));
5250	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
5251	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5252	\|\| !(pCtx->cs.Attr.n.u1Granularity));
5253	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
5254	\|\| (pCtx->cs.Attr.n.u1Granularity));
5255	/* CS cannot be loaded with NULL in protected mode. */
5256	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
5257	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5258	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
5259	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5260	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
5261	else
5262	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
5263	/* SS */
5264	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5265	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
5266	if ( !(pCtx->cr0 & X86_CR0_PE)
5267	\|\| pCtx->cs.Attr.n.u4Type == 3)
5268	{
5269	Assert(!pCtx->ss.Attr.n.u2Dpl);
5270	}
5271	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
5272	{
5273	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5274	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
5275	Assert(pCtx->ss.Attr.n.u1Present);
5276	Assert(!(pCtx->ss.Attr.u & 0xf00));
5277	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
5278	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
5279	\|\| !(pCtx->ss.Attr.n.u1Granularity));
5280	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
5281	\|\| (pCtx->ss.Attr.n.u1Granularity));
5282	}
5283	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
5284	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
5285	{
5286	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5287	Assert(pCtx->ds.Attr.n.u1Present);
5288	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
5289	Assert(!(pCtx->ds.Attr.u & 0xf00));
5290	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
5291	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
5292	\|\| !(pCtx->ds.Attr.n.u1Granularity));
5293	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
5294	\|\| (pCtx->ds.Attr.n.u1Granularity));
5295	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5296	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
5297	}
5298	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
5299	{
5300	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5301	Assert(pCtx->es.Attr.n.u1Present);
5302	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
5303	Assert(!(pCtx->es.Attr.u & 0xf00));
5304	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
5305	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
5306	\|\| !(pCtx->es.Attr.n.u1Granularity));
5307	Assert( !(pCtx->es.u32Limit & 0xfff00000)
5308	\|\| (pCtx->es.Attr.n.u1Granularity));
5309	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5310	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
5311	}
5312	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
5313	{
5314	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5315	Assert(pCtx->fs.Attr.n.u1Present);
5316	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
5317	Assert(!(pCtx->fs.Attr.u & 0xf00));
5318	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
5319	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
5320	\|\| !(pCtx->fs.Attr.n.u1Granularity));
5321	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
5322	\|\| (pCtx->fs.Attr.n.u1Granularity));
5323	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5324	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5325	}
5326	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
5327	{
5328	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5329	Assert(pCtx->gs.Attr.n.u1Present);
5330	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
5331	Assert(!(pCtx->gs.Attr.u & 0xf00));
5332	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
5333	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
5334	\|\| !(pCtx->gs.Attr.n.u1Granularity));
5335	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
5336	\|\| (pCtx->gs.Attr.n.u1Granularity));
5337	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5338	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5339	}
5340	/* 64-bit capable CPUs. */
5341	# if HC_ARCH_BITS == 64
5342	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5343	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
5344	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
5345	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
5346	# endif
5347	}
5348	else if ( CPUMIsGuestInV86ModeEx(pCtx)
5349	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
5350	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
5351	{
5352	/* Real and v86 mode checks. */
5353	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
5354	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
5355	if (pVmcsInfo->RealMode.fRealOnV86Active)
5356	{
5357	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
5358	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
5359	}
5360	else
5361	{
5362	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
5363	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
5364	}
5365
5366	/* CS */
5367	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
5368	Assert(pCtx->cs.u32Limit == 0xffff);
5369	Assert(u32CSAttr == 0xf3);
5370	/* SS */
5371	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
5372	Assert(pCtx->ss.u32Limit == 0xffff);
5373	Assert(u32SSAttr == 0xf3);
5374	/* DS */
5375	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
5376	Assert(pCtx->ds.u32Limit == 0xffff);
5377	Assert(u32DSAttr == 0xf3);
5378	/* ES */
5379	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
5380	Assert(pCtx->es.u32Limit == 0xffff);
5381	Assert(u32ESAttr == 0xf3);
5382	/* FS */
5383	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
5384	Assert(pCtx->fs.u32Limit == 0xffff);
5385	Assert(u32FSAttr == 0xf3);
5386	/* GS */
5387	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
5388	Assert(pCtx->gs.u32Limit == 0xffff);
5389	Assert(u32GSAttr == 0xf3);
5390	/* 64-bit capable CPUs. */
5391	# if HC_ARCH_BITS == 64
5392	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5393	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
5394	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
5395	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
5396	# endif
5397	}
5398	}
5399	#endif /* VBOX_STRICT */
5400
5401
5402	/**
5403	* Exports a guest segment register into the guest-state area in the VMCS.
5404	*
5405	* @returns VBox status code.
5406	* @param pVCpu The cross context virtual CPU structure.
5407	* @param pVmcsInfo The VMCS info. object.
5408	* @param iSegReg The segment register number (X86_SREG_XXX).
5409	* @param pSelReg Pointer to the segment selector.
5410	*
5411	* @remarks No-long-jump zone!!!
5412	*/
5413	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
5414	{
5415	Assert(iSegReg < X86_SREG_COUNT);
5416	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
5417	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
5418	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
5419	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
5420
5421	uint32_t u32Access = pSelReg->Attr.u;
5422	if (pVmcsInfo->RealMode.fRealOnV86Active)
5423	{
5424	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
5425	u32Access = 0xf3;
5426	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5427	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5428	RT_NOREF_PV(pVCpu);
5429	}
5430	else
5431	{
5432	/*
5433	* The way to differentiate between whether this is really a null selector or was just
5434	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
5435	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
5436	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
5437	* NULL selectors loaded in protected-mode have their attribute as 0.
5438	*/
5439	if (!u32Access)
5440	u32Access = X86DESCATTR_UNUSABLE;
5441	}
5442
5443	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
5444	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
5445	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
5446
5447	/*
5448	* Commit it to the VMCS.
5449	*/
5450	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
5451	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
5452	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
5453	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
5454	AssertRCReturn(rc, rc);
5455	return rc;
5456	}
5457
5458
5459	/**
5460	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
5461	* area in the VMCS.
5462	*
5463	* @returns VBox status code.
5464	* @param pVCpu The cross context virtual CPU structure.
5465	* @param pVmxTransient The VMX-transient structure.
5466	*
5467	* @remarks Will import guest CR0 on strict builds during validation of
5468	* segments.
5469	* @remarks No-long-jump zone!!!
5470	*/
5471	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5472	{
5473	int rc = VERR_INTERNAL_ERROR_5;
5474	PVM pVM = pVCpu->CTX_SUFF(pVM);
5475	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5476	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5477
5478	/*
5479	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
5480	*/
5481	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
5482	{
5483	#ifdef VBOX_WITH_REM
5484	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
5485	{
5486	Assert(!pVmxTransient->fIsNestedGuest);
5487	Assert(pVM->hm.s.vmx.pRealModeTSS);
5488	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
5489	if ( pVmcsInfo->fWasInRealMode
5490	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
5491	{
5492	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
5493	in real-mode (e.g. OpenBSD 4.0) */
5494	REMFlushTBs(pVM);
5495	Log4Func(("Switch to protected mode detected!\n"));
5496	pVmcsInfo->fWasInRealMode = false;
5497	}
5498	}
5499	#endif
5500	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
5501	{
5502	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
5503	if (pVmcsInfo->RealMode.fRealOnV86Active)
5504	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
5505	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
5506	AssertRCReturn(rc, rc);
5507	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
5508	}
5509
5510	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
5511	{
5512	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
5513	if (pVmcsInfo->RealMode.fRealOnV86Active)
5514	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
5515	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
5516	AssertRCReturn(rc, rc);
5517	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
5518	}
5519
5520	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
5521	{
5522	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
5523	if (pVmcsInfo->RealMode.fRealOnV86Active)
5524	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
5525	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
5526	AssertRCReturn(rc, rc);
5527	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
5528	}
5529
5530	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
5531	{
5532	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
5533	if (pVmcsInfo->RealMode.fRealOnV86Active)
5534	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
5535	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
5536	AssertRCReturn(rc, rc);
5537	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
5538	}
5539
5540	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
5541	{
5542	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
5543	if (pVmcsInfo->RealMode.fRealOnV86Active)
5544	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
5545	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
5546	AssertRCReturn(rc, rc);
5547	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
5548	}
5549
5550	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
5551	{
5552	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
5553	if (pVmcsInfo->RealMode.fRealOnV86Active)
5554	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
5555	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
5556	AssertRCReturn(rc, rc);
5557	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
5558	}
5559
5560	#ifdef VBOX_STRICT
5561	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
5562	#endif
5563	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
5564	pCtx->cs.Attr.u));
5565	}
5566
5567	/*
5568	* Guest TR.
5569	*/
5570	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
5571	{
5572	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
5573
5574	/*
5575	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
5576	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
5577	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
5578	*/
5579	uint16_t u16Sel;
5580	uint32_t u32Limit;
5581	uint64_t u64Base;
5582	uint32_t u32AccessRights;
5583	if (!pVmcsInfo->RealMode.fRealOnV86Active)
5584	{
5585	u16Sel = pCtx->tr.Sel;
5586	u32Limit = pCtx->tr.u32Limit;
5587	u64Base = pCtx->tr.u64Base;
5588	u32AccessRights = pCtx->tr.Attr.u;
5589	}
5590	else
5591	{
5592	Assert(!pVmxTransient->fIsNestedGuest);
5593	Assert(pVM->hm.s.vmx.pRealModeTSS);
5594	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
5595
5596	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
5597	RTGCPHYS GCPhys;
5598	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
5599	AssertRCReturn(rc, rc);
5600
5601	X86DESCATTR DescAttr;
5602	DescAttr.u = 0;
5603	DescAttr.n.u1Present = 1;
5604	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
5605
5606	u16Sel = 0;
5607	u32Limit = HM_VTX_TSS_SIZE;
5608	u64Base = GCPhys;
5609	u32AccessRights = DescAttr.u;
5610	}
5611
5612	/* Validate. */
5613	Assert(!(u16Sel & RT_BIT(2)));
5614	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
5615	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
5616	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
5617	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
5618	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
5619	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
5620	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
5621	Assert( (u32Limit & 0xfff) == 0xfff
5622	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
5623	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
5624	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
5625
5626	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
5627	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
5628	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
5629	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
5630	AssertRCReturn(rc, rc);
5631
5632	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
5633	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
5634	}
5635
5636	/*
5637	* Guest GDTR.
5638	*/
5639	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
5640	{
5641	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
5642
5643	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
5644	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
5645	AssertRCReturn(rc, rc);
5646
5647	/* Validate. */
5648	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5649
5650	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
5651	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
5652	}
5653
5654	/*
5655	* Guest LDTR.
5656	*/
5657	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
5658	{
5659	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
5660
5661	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
5662	uint32_t u32Access;
5663	if ( !pVmxTransient->fIsNestedGuest
5664	&& !pCtx->ldtr.Attr.u)
5665	u32Access = X86DESCATTR_UNUSABLE;
5666	else
5667	u32Access = pCtx->ldtr.Attr.u;
5668
5669	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
5670	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
5671	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
5672	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
5673	AssertRCReturn(rc, rc);
5674
5675	/* Validate. */
5676	if (!(u32Access & X86DESCATTR_UNUSABLE))
5677	{
5678	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
5679	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
5680	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
5681	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
5682	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
5683	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
5684	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
5685	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
5686	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
5687	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
5688	}
5689
5690	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
5691	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
5692	}
5693
5694	/*
5695	* Guest IDTR.
5696	*/
5697	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
5698	{
5699	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
5700
5701	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
5702	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
5703	AssertRCReturn(rc, rc);
5704
5705	/* Validate. */
5706	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5707
5708	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
5709	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
5710	}
5711
5712	return VINF_SUCCESS;
5713	}
5714
5715
5716	/**
5717	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
5718	* areas.
5719	*
5720	* These MSRs will automatically be loaded to the host CPU on every successful
5721	* VM-entry and stored from the host CPU on every successful VM-exit.
5722	*
5723	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
5724	* actual host MSR values are not- updated here for performance reasons. See
5725	* hmR0VmxExportHostMsrs().
5726	*
5727	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
5728	*
5729	* @returns VBox status code.
5730	* @param pVCpu The cross context virtual CPU structure.
5731	* @param pVmxTransient The VMX-transient structure.
5732	*
5733	* @remarks No-long-jump zone!!!
5734	*/
5735	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5736	{
5737	AssertPtr(pVCpu);
5738	AssertPtr(pVmxTransient);
5739
5740	PVM pVM = pVCpu->CTX_SUFF(pVM);
5741	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5742
5743	/*
5744	* MSRs that we use the auto-load/store MSR area in the VMCS.
5745	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
5746	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
5747	*
5748	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
5749	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
5750	* emulation, nothing to do here.
5751	*/
5752	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
5753	{
5754	if ( !pVmxTransient->fIsNestedGuest
5755	&& pVM->hm.s.fAllow64BitGuests)
5756	{
5757	#if HC_ARCH_BITS == 32
5758	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
5759	Assert(!pVmxTransient->fIsNestedGuest);
5760
5761	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
5762	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
5763	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
5764	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
5765	AssertRCReturn(rc, rc);
5766	#endif
5767	}
5768	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
5769	}
5770
5771	/*
5772	* Guest Sysenter MSRs.
5773	*/
5774	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
5775	{
5776	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
5777
5778	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
5779	{
5780	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
5781	AssertRCReturn(rc, rc);
5782	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
5783	}
5784
5785	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
5786	{
5787	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
5788	AssertRCReturn(rc, rc);
5789	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
5790	}
5791
5792	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
5793	{
5794	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
5795	AssertRCReturn(rc, rc);
5796	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
5797	}
5798	}
5799
5800	/*
5801	* Guest/host EFER MSR.
5802	*/
5803	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
5804	{
5805	/* Whether we are using the VMCS to swap the EFER MSR must have been
5806	determined earlier while exporting VM-entry/VM-exit controls. */
5807	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
5808	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
5809
5810	if (hmR0VmxShouldSwapEferMsr(pVCpu))
5811	{
5812	/*
5813	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
5814	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
5815	*/
5816	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
5817	{
5818	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
5819	AssertRCReturn(rc, rc);
5820	}
5821	else
5822	{
5823	/*
5824	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
5825	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
5826	*/
5827	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
5828	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5829	AssertRCReturn(rc, rc);
5830	}
5831	}
5832	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
5833	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
5834
5835	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
5836	}
5837
5838	/*
5839	* Other MSRs.
5840	* Speculation Control (R/W).
5841	*/
5842	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
5843	{
5844	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
5845	if (pVM->cpum.ro.GuestFeatures.fIbrs)
5846	{
5847	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
5848	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5849	AssertRCReturn(rc, rc);
5850	}
5851	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
5852	}
5853
5854	return VINF_SUCCESS;
5855	}
5856
5857
5858	/**
5859	* Selects up the appropriate function to run guest code.
5860	*
5861	* @returns VBox status code.
5862	* @param pVCpu The cross context virtual CPU structure.
5863	* @param pVmcsInfo The VMCS info. object.
5864	*
5865	* @remarks No-long-jump zone!!!
5866	*/
5867	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5868	{
5869	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5870	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5871
5872	if (CPUMIsGuestInLongModeEx(pCtx))
5873	{
5874	#ifndef VBOX_ENABLE_64_BITS_GUESTS
5875	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5876	#endif
5877	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
5878	#if HC_ARCH_BITS == 32
5879	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
5880	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
5881	{
5882	#ifdef VBOX_STRICT
5883	if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5884	{
5885	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5886	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5887	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5888	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5889	("fCtxChanged=%#RX64\n", fCtxChanged));
5890	}
5891	#endif
5892	pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
5893
5894	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
5895	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
5896	pVmcsInfo->fSwitchedTo64on32 = true;
5897	Log4Func(("Selected 64-bit switcher\n"));
5898	}
5899	#else
5900	/* 64-bit host. */
5901	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
5902	#endif
5903	}
5904	else
5905	{
5906	/* Guest is not in long mode, use the 32-bit handler. */
5907	#if HC_ARCH_BITS == 32
5908	if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
5909	&& !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
5910	&& pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5911	{
5912	# ifdef VBOX_STRICT
5913	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5914	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5915	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5916	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5917	("fCtxChanged=%#RX64\n", fCtxChanged));
5918	# endif
5919	}
5920	# ifdef VBOX_ENABLE_64_BITS_GUESTS
5921	/*
5922	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
5923	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
5924	* switcher flag now because we know the guest is in a sane state where it's safe
5925	* to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
5926	* the much faster 32-bit switcher again.
5927	*/
5928	if (!pVmcsInfo->fSwitchedTo64on32)
5929	{
5930	if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
5931	Log4Func(("Selected 32-bit switcher\n"));
5932	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5933	}
5934	else
5935	{
5936	Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
5937	if ( pVmcsInfo->RealMode.fRealOnV86Active
5938	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
5939	{
5940	pVmcsInfo->fSwitchedTo64on32 = false;
5941	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5942	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
5943	\| HM_CHANGED_VMX_ENTRY_EXIT_CTLS
5944	\| HM_CHANGED_HOST_CONTEXT);
5945	Log4Func(("Selected 32-bit switcher (safe)\n"));
5946	}
5947	}
5948	# else
5949	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5950	# endif
5951	#else
5952	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5953	#endif
5954	}
5955	Assert(pVmcsInfo->pfnStartVM);
5956	return VINF_SUCCESS;
5957	}
5958
5959
5960	/**
5961	* Wrapper for running the guest code in VT-x.
5962	*
5963	* @returns VBox status code, no informational status codes.
5964	* @param pVCpu The cross context virtual CPU structure.
5965	* @param pVmxTransient The VMX-transient structure.
5966	*
5967	* @remarks No-long-jump zone!!!
5968	*/
5969	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5970	{
5971	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
5972	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5973	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
5974
5975	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
5976
5977	/*
5978	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
5979	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
5980	* callee-saved and thus the need for this XMM wrapper.
5981	*
5982	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
5983	*/
5984	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5985	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
5986	PVM pVM = pVCpu->CTX_SUFF(pVM);
5987	#ifdef VBOX_WITH_KERNEL_USING_XMM
5988	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
5989	#else
5990	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
5991	#endif
5992	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5993	return rc;
5994	}
5995
5996
5997	/**
5998	* Reports world-switch error and dumps some useful debug info.
5999	*
6000	* @param pVCpu The cross context virtual CPU structure.
6001	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6002	* @param pVmxTransient The VMX-transient structure (only
6003	* exitReason updated).
6004	*/
6005	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6006	{
6007	Assert(pVCpu);
6008	Assert(pVmxTransient);
6009	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6010
6011	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6012	switch (rcVMRun)
6013	{
6014	case VERR_VMX_INVALID_VMXON_PTR:
6015	AssertFailed();
6016	break;
6017	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6018	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6019	{
6020	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6021	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6022	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
6023	AssertRC(rc);
6024
6025	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6026	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6027	Cannot do it here as we may have been long preempted. */
6028
6029	#ifdef VBOX_STRICT
6030	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6031	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6032	pVmxTransient->uExitReason));
6033	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6034	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6035	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6036	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6037	else
6038	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6039	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6040	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6041
6042	/* VMX control bits. */
6043	uint32_t u32Val;
6044	uint64_t u64Val;
6045	RTHCUINTREG uHCReg;
6046	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
6047	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
6048	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
6049	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
6050	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
6051	{
6052	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
6053	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
6054	}
6055	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
6056	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
6057	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
6058	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
6059	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
6060	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
6061	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
6062	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
6063	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
6064	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
6065	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
6066	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
6067	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
6068	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
6069	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
6070	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
6071	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6072	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
6073	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6074	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
6075	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
6076	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
6077	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
6078	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
6079	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
6080	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
6081	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
6082	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
6083	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
6084	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6085	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
6086	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
6087	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
6088	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6089	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6090	{
6091	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
6092	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
6093	}
6094
6095	/* Guest bits. */
6096	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
6097	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
6098	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
6099	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
6100	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
6101	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
6102	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
6103	{
6104	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
6105	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
6106	}
6107
6108	/* Host bits. */
6109	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
6110	Log4(("Host CR0 %#RHr\n", uHCReg));
6111	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
6112	Log4(("Host CR3 %#RHr\n", uHCReg));
6113	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
6114	Log4(("Host CR4 %#RHr\n", uHCReg));
6115
6116	RTGDTR HostGdtr;
6117	PCX86DESCHC pDesc;
6118	ASMGetGDTR(&HostGdtr);
6119	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
6120	Log4(("Host CS %#08x\n", u32Val));
6121	if (u32Val < HostGdtr.cbGdt)
6122	{
6123	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6124	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
6125	}
6126
6127	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
6128	Log4(("Host DS %#08x\n", u32Val));
6129	if (u32Val < HostGdtr.cbGdt)
6130	{
6131	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6132	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
6133	}
6134
6135	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
6136	Log4(("Host ES %#08x\n", u32Val));
6137	if (u32Val < HostGdtr.cbGdt)
6138	{
6139	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6140	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
6141	}
6142
6143	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
6144	Log4(("Host FS %#08x\n", u32Val));
6145	if (u32Val < HostGdtr.cbGdt)
6146	{
6147	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6148	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
6149	}
6150
6151	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
6152	Log4(("Host GS %#08x\n", u32Val));
6153	if (u32Val < HostGdtr.cbGdt)
6154	{
6155	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6156	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
6157	}
6158
6159	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
6160	Log4(("Host SS %#08x\n", u32Val));
6161	if (u32Val < HostGdtr.cbGdt)
6162	{
6163	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6164	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
6165	}
6166
6167	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
6168	Log4(("Host TR %#08x\n", u32Val));
6169	if (u32Val < HostGdtr.cbGdt)
6170	{
6171	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6172	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
6173	}
6174
6175	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
6176	Log4(("Host TR Base %#RHv\n", uHCReg));
6177	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
6178	Log4(("Host GDTR Base %#RHv\n", uHCReg));
6179	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
6180	Log4(("Host IDTR Base %#RHv\n", uHCReg));
6181	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
6182	Log4(("Host SYSENTER CS %#08x\n", u32Val));
6183	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
6184	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
6185	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
6186	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
6187	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
6188	Log4(("Host RSP %#RHv\n", uHCReg));
6189	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
6190	Log4(("Host RIP %#RHv\n", uHCReg));
6191	# if HC_ARCH_BITS == 64
6192	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6193	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6194	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6195	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6196	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6197	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6198	# endif
6199	#endif /* VBOX_STRICT */
6200	break;
6201	}
6202
6203	default:
6204	/* Impossible */
6205	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6206	break;
6207	}
6208	}
6209
6210
6211	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
6212	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
6213	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
6214	#endif
6215	#ifdef VBOX_STRICT
6216	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
6217	{
6218	switch (idxField)
6219	{
6220	case VMX_VMCS_GUEST_RIP:
6221	case VMX_VMCS_GUEST_RSP:
6222	case VMX_VMCS_GUEST_SYSENTER_EIP:
6223	case VMX_VMCS_GUEST_SYSENTER_ESP:
6224	case VMX_VMCS_GUEST_GDTR_BASE:
6225	case VMX_VMCS_GUEST_IDTR_BASE:
6226	case VMX_VMCS_GUEST_CS_BASE:
6227	case VMX_VMCS_GUEST_DS_BASE:
6228	case VMX_VMCS_GUEST_ES_BASE:
6229	case VMX_VMCS_GUEST_FS_BASE:
6230	case VMX_VMCS_GUEST_GS_BASE:
6231	case VMX_VMCS_GUEST_SS_BASE:
6232	case VMX_VMCS_GUEST_LDTR_BASE:
6233	case VMX_VMCS_GUEST_TR_BASE:
6234	case VMX_VMCS_GUEST_CR3:
6235	return true;
6236	}
6237	return false;
6238	}
6239
6240	static bool hmR0VmxIsValidReadField(uint32_t idxField)
6241	{
6242	switch (idxField)
6243	{
6244	/* Read-only fields. */
6245	case VMX_VMCS_RO_EXIT_QUALIFICATION:
6246	return true;
6247	}
6248	/* Remaining readable fields should also be writable. */
6249	return hmR0VmxIsValidWriteField(idxField);
6250	}
6251	#endif /* VBOX_STRICT */
6252
6253
6254	/**
6255	* Initialize the VMCS-Read cache.
6256	*
6257	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
6258	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
6259	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
6260	* (those that have a 32-bit FULL & HIGH part).
6261	*
6262	* @param pVCpu The cross context virtual CPU structure.
6263	*/
6264	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
6265	{
6266	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
6267	do { \
6268	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
6269	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
6270	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
6271	++cReadFields; \
6272	} while (0)
6273
6274	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6275	uint32_t cReadFields = 0;
6276
6277	/*
6278	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
6279	* and serve to indicate exceptions to the rules.
6280	*/
6281
6282	/* Guest-natural selector base fields. */
6283	#if 0
6284	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
6285	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
6286	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
6287	#endif
6288	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
6289	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
6290	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
6291	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
6292	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
6293	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
6294	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
6295	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
6296	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
6297	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
6298	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
6299	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
6300	#if 0
6301	/* Unused natural width guest-state fields. */
6302	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
6303	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
6304	#endif
6305	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
6306	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
6307
6308	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
6309	these 64-bit fields (using "FULL" and "HIGH" fields). */
6310	#if 0
6311	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
6312	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
6313	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
6314	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
6315	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
6316	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
6317	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
6318	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
6319	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
6320	#endif
6321
6322	/* Natural width guest-state fields. */
6323	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
6324	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
6325
6326	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6327	{
6328	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
6329	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
6330	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
6331	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
6332	}
6333	else
6334	{
6335	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
6336	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
6337	}
6338
6339	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
6340	}
6341
6342
6343	/**
6344	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
6345	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
6346	* darwin, running 64-bit guests).
6347	*
6348	* @returns VBox status code.
6349	* @param pVCpu The cross context virtual CPU structure.
6350	* @param idxField The VMCS field encoding.
6351	* @param u64Val 16, 32 or 64-bit value.
6352	*/
6353	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6354	{
6355	int rc;
6356	switch (idxField)
6357	{
6358	/*
6359	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
6360	*/
6361	/* 64-bit Control fields. */
6362	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
6363	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
6364	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
6365	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
6366	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
6367	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
6368	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
6369	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
6370	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
6371	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
6372	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
6373	case VMX_VMCS64_CTRL_EPTP_FULL:
6374	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
6375	/* 64-bit Guest-state fields. */
6376	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
6377	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
6378	case VMX_VMCS64_GUEST_PAT_FULL:
6379	case VMX_VMCS64_GUEST_EFER_FULL:
6380	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
6381	case VMX_VMCS64_GUEST_PDPTE0_FULL:
6382	case VMX_VMCS64_GUEST_PDPTE1_FULL:
6383	case VMX_VMCS64_GUEST_PDPTE2_FULL:
6384	case VMX_VMCS64_GUEST_PDPTE3_FULL:
6385	/* 64-bit Host-state fields. */
6386	case VMX_VMCS64_HOST_PAT_FULL:
6387	case VMX_VMCS64_HOST_EFER_FULL:
6388	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
6389	{
6390	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6391	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
6392	break;
6393	}
6394
6395	/*
6396	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
6397	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
6398	*/
6399	/* Natural-width Guest-state fields. */
6400	case VMX_VMCS_GUEST_CR3:
6401	case VMX_VMCS_GUEST_ES_BASE:
6402	case VMX_VMCS_GUEST_CS_BASE:
6403	case VMX_VMCS_GUEST_SS_BASE:
6404	case VMX_VMCS_GUEST_DS_BASE:
6405	case VMX_VMCS_GUEST_FS_BASE:
6406	case VMX_VMCS_GUEST_GS_BASE:
6407	case VMX_VMCS_GUEST_LDTR_BASE:
6408	case VMX_VMCS_GUEST_TR_BASE:
6409	case VMX_VMCS_GUEST_GDTR_BASE:
6410	case VMX_VMCS_GUEST_IDTR_BASE:
6411	case VMX_VMCS_GUEST_RSP:
6412	case VMX_VMCS_GUEST_RIP:
6413	case VMX_VMCS_GUEST_SYSENTER_ESP:
6414	case VMX_VMCS_GUEST_SYSENTER_EIP:
6415	{
6416	if (!(RT_HI_U32(u64Val)))
6417	{
6418	/* If this field is 64-bit, VT-x will zero out the top bits. */
6419	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6420	}
6421	else
6422	{
6423	/* Assert that only the 32->64 switcher case should ever come here. */
6424	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
6425	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
6426	}
6427	break;
6428	}
6429
6430	default:
6431	{
6432	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
6433	rc = VERR_INVALID_PARAMETER;
6434	break;
6435	}
6436	}
6437	AssertRCReturn(rc, rc);
6438	return rc;
6439	}
6440
6441
6442	/**
6443	* Queue up a VMWRITE by using the VMCS write cache.
6444	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
6445	*
6446	* @param pVCpu The cross context virtual CPU structure.
6447	* @param idxField The VMCS field encoding.
6448	* @param u64Val 16, 32 or 64-bit value.
6449	*/
6450	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6451	{
6452	AssertPtr(pVCpu);
6453	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6454
6455	AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
6456	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
6457
6458	/* Make sure there are no duplicates. */
6459	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
6460	{
6461	if (pCache->Write.aField[i] == idxField)
6462	{
6463	pCache->Write.aFieldVal[i] = u64Val;
6464	return VINF_SUCCESS;
6465	}
6466	}
6467
6468	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
6469	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
6470	pCache->Write.cValidEntries++;
6471	return VINF_SUCCESS;
6472	}
6473	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
6474
6475
6476	/**
6477	* Sets up the usage of TSC-offsetting and updates the VMCS.
6478	*
6479	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6480	* VMX-preemption timer.
6481	*
6482	* @returns VBox status code.
6483	* @param pVCpu The cross context virtual CPU structure.
6484	* @param pVmxTransient The VMX-transient structure.
6485	*
6486	* @remarks No-long-jump zone!!!
6487	*/
6488	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6489	{
6490	bool fOffsettedTsc;
6491	bool fParavirtTsc;
6492	uint64_t uTscOffset;
6493	PVM pVM = pVCpu->CTX_SUFF(pVM);
6494	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);;
6495
6496	if (pVM->hm.s.vmx.fUsePreemptTimer)
6497	{
6498	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6499
6500	/* Make sure the returned values have sane upper and lower boundaries. */
6501	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6502	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6503	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6504	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6505
6506	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6507	* preemption timers here. We probably need to clamp the preemption timer,
6508	* after converting the timer value to the host. */
6509	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6510	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6511	AssertRC(rc);
6512	}
6513	else
6514	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6515
6516	if (fParavirtTsc)
6517	{
6518	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6519	information before every VM-entry, hence disable it for performance sake. */
6520	#if 0
6521	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
6522	AssertRC(rc);
6523	#endif
6524	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
6525	}
6526
6527	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
6528	if ( fOffsettedTsc
6529	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
6530	{
6531	if (pVmxTransient->fIsNestedGuest)
6532	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
6533	if (pVmcsInfo->u64TscOffset != uTscOffset)
6534	{
6535	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
6536	AssertRC(rc);
6537	pVmcsInfo->u64TscOffset = uTscOffset;
6538	}
6539
6540	if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
6541	{
6542	uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
6543	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6544	AssertRC(rc);
6545	pVmcsInfo->u32ProcCtls = uProcCtls;
6546	}
6547	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
6548	}
6549	else
6550	{
6551	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
6552	if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
6553	{
6554	uProcCtls \|= VMX_PROC_CTLS_RDTSC_EXIT;
6555	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6556	AssertRC(rc);
6557	pVmcsInfo->u32ProcCtls = uProcCtls;
6558	}
6559	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
6560	}
6561	}
6562
6563
6564	/**
6565	* Gets the IEM exception flags for the specified vector and IDT vectoring /
6566	* VM-exit interruption info type.
6567	*
6568	* @returns The IEM exception flags.
6569	* @param uVector The event vector.
6570	* @param uVmxEventType The VMX event type.
6571	*
6572	* @remarks This function currently only constructs flags required for
6573	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
6574	* and CR2 aspects of an exception are not included).
6575	*/
6576	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
6577	{
6578	uint32_t fIemXcptFlags;
6579	switch (uVmxEventType)
6580	{
6581	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6582	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6583	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6584	break;
6585
6586	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6587	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6588	break;
6589
6590	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6591	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
6592	break;
6593
6594	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
6595	{
6596	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6597	if (uVector == X86_XCPT_BP)
6598	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
6599	else if (uVector == X86_XCPT_OF)
6600	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
6601	else
6602	{
6603	fIemXcptFlags = 0;
6604	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
6605	}
6606	break;
6607	}
6608
6609	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6610	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6611	break;
6612
6613	default:
6614	fIemXcptFlags = 0;
6615	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
6616	break;
6617	}
6618	return fIemXcptFlags;
6619	}
6620
6621
6622	/**
6623	* Sets an event as a pending event to be injected into the guest.
6624	*
6625	* @param pVCpu The cross context virtual CPU structure.
6626	* @param u32IntInfo The VM-entry interruption-information field.
6627	* @param cbInstr The VM-entry instruction length in bytes (for software
6628	* interrupts, exceptions and privileged software
6629	* exceptions).
6630	* @param u32ErrCode The VM-entry exception error code.
6631	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
6632	* page-fault.
6633	*/
6634	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
6635	RTGCUINTPTR GCPtrFaultAddress)
6636	{
6637	Assert(!pVCpu->hm.s.Event.fPending);
6638	pVCpu->hm.s.Event.fPending = true;
6639	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
6640	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
6641	pVCpu->hm.s.Event.cbInstr = cbInstr;
6642	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
6643	}
6644
6645
6646	/**
6647	* Sets an external interrupt as pending-for-injection into the VM.
6648	*
6649	* @param pVCpu The cross context virtual CPU structure.
6650	* @param u8Interrupt The external interrupt vector.
6651	*/
6652	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
6653	{
6654	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
6655	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6656	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6657	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6658	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6659	}
6660
6661
6662	/**
6663	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
6664	*
6665	* @param pVCpu The cross context virtual CPU structure.
6666	*/
6667	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
6668	{
6669	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
6670	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
6671	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6672	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6673	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6674	}
6675
6676
6677	/**
6678	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
6679	*
6680	* @param pVCpu The cross context virtual CPU structure.
6681	*/
6682	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
6683	{
6684	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
6685	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6686	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6687	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6688	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6689	}
6690
6691
6692	/**
6693	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
6694	*
6695	* @param pVCpu The cross context virtual CPU structure.
6696	*/
6697	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
6698	{
6699	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
6700	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6701	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6702	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6703	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6704	}
6705
6706
6707	/**
6708	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
6709	*
6710	* @param pVCpu The cross context virtual CPU structure.
6711	*/
6712	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
6713	{
6714	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
6715	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6716	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6717	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6718	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6719	}
6720
6721
6722	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6723	/**
6724	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
6725	*
6726	* @param pVCpu The cross context virtual CPU structure.
6727	* @param u32ErrCode The error code for the general-protection exception.
6728	*/
6729	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
6730	{
6731	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
6732	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6733	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6734	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6735	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6736	}
6737
6738
6739	/**
6740	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
6741	*
6742	* @param pVCpu The cross context virtual CPU structure.
6743	* @param u32ErrCode The error code for the stack exception.
6744	*/
6745	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
6746	{
6747	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
6748	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6749	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6750	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6751	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6752	}
6753
6754
6755	/**
6756	* Decodes the memory operand of an instruction that caused a VM-exit.
6757	*
6758	* The VM-exit qualification field provides the displacement field for memory
6759	* operand instructions, if any.
6760	*
6761	* @returns Strict VBox status code (i.e. informational status codes too).
6762	* @retval VINF_SUCCESS if the operand was successfully decoded.
6763	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
6764	* operand.
6765	* @param pVCpu The cross context virtual CPU structure.
6766	* @param uExitInstrInfo The VM-exit instruction information field.
6767	* @param enmMemAccess The memory operand's access type (read or write).
6768	* @param GCPtrDisp The instruction displacement field, if any. For
6769	* RIP-relative addressing pass RIP + displacement here.
6770	* @param pGCPtrMem Where to store the effective destination memory address.
6771	*/
6772	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
6773	PRTGCPTR pGCPtrMem)
6774	{
6775	Assert(pGCPtrMem);
6776	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
6777	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
6778	\| CPUMCTX_EXTRN_CR0);
6779
6780	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
6781	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
6782	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
6783
6784	VMXEXITINSTRINFO ExitInstrInfo;
6785	ExitInstrInfo.u = uExitInstrInfo;
6786	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
6787	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
6788	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
6789	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
6790	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
6791	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
6792	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
6793	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
6794	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
6795
6796	/*
6797	* Validate instruction information.
6798	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
6799	*/
6800	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
6801	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
6802	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
6803	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
6804	AssertLogRelMsgReturn(fIsMemOperand,
6805	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
6806
6807	/*
6808	* Compute the complete effective address.
6809	*
6810	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
6811	* See AMD spec. 4.5.2 "Segment Registers".
6812	*/
6813	RTGCPTR GCPtrMem = GCPtrDisp;
6814	if (fBaseRegValid)
6815	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
6816	if (fIdxRegValid)
6817	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
6818
6819	RTGCPTR const GCPtrOff = GCPtrMem;
6820	if ( !fIsLongMode
6821	\|\| iSegReg >= X86_SREG_FS)
6822	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
6823	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
6824
6825	/*
6826	* Validate effective address.
6827	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
6828	*/
6829	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
6830	Assert(cbAccess > 0);
6831	if (fIsLongMode)
6832	{
6833	if (X86_IS_CANONICAL(GCPtrMem))
6834	{
6835	*pGCPtrMem = GCPtrMem;
6836	return VINF_SUCCESS;
6837	}
6838
6839	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
6840	* "Data Limit Checks in 64-bit Mode". */
6841	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
6842	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6843	return VINF_HM_PENDING_XCPT;
6844	}
6845
6846	/*
6847	* This is a watered down version of iemMemApplySegment().
6848	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
6849	* and segment CPL/DPL checks are skipped.
6850	*/
6851	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
6852	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
6853	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
6854
6855	/* Check if the segment is present and usable. */
6856	if ( pSel->Attr.n.u1Present
6857	&& !pSel->Attr.n.u1Unusable)
6858	{
6859	Assert(pSel->Attr.n.u1DescType);
6860	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
6861	{
6862	/* Check permissions for the data segment. */
6863	if ( enmMemAccess == VMXMEMACCESS_WRITE
6864	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
6865	{
6866	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6867	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
6868	return VINF_HM_PENDING_XCPT;
6869	}
6870
6871	/* Check limits if it's a normal data segment. */
6872	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
6873	{
6874	if ( GCPtrFirst32 > pSel->u32Limit
6875	\|\| GCPtrLast32 > pSel->u32Limit)
6876	{
6877	Log4Func(("Data segment limit exceeded."
6878	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6879	GCPtrLast32, pSel->u32Limit));
6880	if (iSegReg == X86_SREG_SS)
6881	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6882	else
6883	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6884	return VINF_HM_PENDING_XCPT;
6885	}
6886	}
6887	else
6888	{
6889	/* Check limits if it's an expand-down data segment.
6890	Note! The upper boundary is defined by the B bit, not the G bit! */
6891	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
6892	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
6893	{
6894	Log4Func(("Expand-down data segment limit exceeded."
6895	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6896	GCPtrLast32, pSel->u32Limit));
6897	if (iSegReg == X86_SREG_SS)
6898	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6899	else
6900	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6901	return VINF_HM_PENDING_XCPT;
6902	}
6903	}
6904	}
6905	else
6906	{
6907	/* Check permissions for the code segment. */
6908	if ( enmMemAccess == VMXMEMACCESS_WRITE
6909	\|\| ( enmMemAccess == VMXMEMACCESS_READ
6910	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
6911	{
6912	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6913	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6914	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6915	return VINF_HM_PENDING_XCPT;
6916	}
6917
6918	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6919	if ( GCPtrFirst32 > pSel->u32Limit
6920	\|\| GCPtrLast32 > pSel->u32Limit)
6921	{
6922	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6923	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6924	if (iSegReg == X86_SREG_SS)
6925	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6926	else
6927	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6928	return VINF_HM_PENDING_XCPT;
6929	}
6930	}
6931	}
6932	else
6933	{
6934	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6935	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6936	return VINF_HM_PENDING_XCPT;
6937	}
6938
6939	*pGCPtrMem = GCPtrMem;
6940	return VINF_SUCCESS;
6941	}
6942
6943
6944	/**
6945	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6946	* guest attempting to execute a VMX instruction.
6947	*
6948	* @returns Strict VBox status code (i.e. informational status codes too).
6949	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
6950	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
6951	*
6952	* @param pVCpu The cross context virtual CPU structure.
6953	* @param uExitReason The VM-exit reason.
6954	*
6955	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
6956	* @remarks No-long-jump zone!!!
6957	*/
6958	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
6959	{
6960	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
6961	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
6962
6963	if ( CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx)
6964	\|\| ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6965	&& !CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)))
6966	{
6967	Log4Func(("In real/v86-mode or long-mode outside 64-bit code segment -> #UD\n"));
6968	hmR0VmxSetPendingXcptUD(pVCpu);
6969	return VINF_HM_PENDING_XCPT;
6970	}
6971
6972	if (uExitReason == VMX_EXIT_VMXON)
6973	{
6974	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
6975
6976	/*
6977	* We check CR4.VMXE because it is required to be always set while in VMX operation
6978	* by physical CPUs and our CR4 read shadow is only consulted when executing specific
6979	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
6980	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
6981	*/
6982	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
6983	{
6984	Log4Func(("CR4.VMXE is not set -> #UD\n"));
6985	hmR0VmxSetPendingXcptUD(pVCpu);
6986	return VINF_HM_PENDING_XCPT;
6987	}
6988	}
6989	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
6990	{
6991	/*
6992	* The guest has not entered VMX operation but attempted to execute a VMX instruction
6993	* (other than VMXON), we need to raise a #UD.
6994	*/
6995	Log4Func(("Not in VMX root mode -> #UD\n"));
6996	hmR0VmxSetPendingXcptUD(pVCpu);
6997	return VINF_HM_PENDING_XCPT;
6998	}
6999
7000	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
7001	{
7002	/*
7003	* The nested-guest attempted to execute a VMX instruction, cause a VM-exit and let
7004	* the guest hypervisor deal with it.
7005	*/
7006	/** @todo NSTVMX: Trigger a VM-exit */
7007	}
7008
7009	/*
7010	* VMX instructions require CPL 0 except in VMX non-root mode where the VM-exit intercept
7011	* (above) takes preceedence over the CPL check.
7012	*/
7013	if (CPUMGetGuestCPL(pVCpu) > 0)
7014	{
7015	Log4Func(("CPL > 0 -> #GP(0)\n"));
7016	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7017	return VINF_HM_PENDING_XCPT;
7018	}
7019
7020	return VINF_SUCCESS;
7021	}
7022	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7023
7024
7025	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7026	{
7027	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7028
7029	/*
7030	* If VT-x marks the segment as unusable, most other bits remain undefined:
7031	* - For CS the L, D and G bits have meaning.
7032	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7033	* - For the remaining data segments no bits are defined.
7034	*
7035	* The present bit and the unusable bit has been observed to be set at the
7036	* same time (the selector was supposed to be invalid as we started executing
7037	* a V8086 interrupt in ring-0).
7038	*
7039	* What should be important for the rest of the VBox code, is that the P bit is
7040	* cleared. Some of the other VBox code recognizes the unusable bit, but
7041	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7042	* safe side here, we'll strip off P and other bits we don't care about. If
7043	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7044	*
7045	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7046	*/
7047	#ifdef VBOX_STRICT
7048	uint32_t const uAttr = pSelReg->Attr.u;
7049	#endif
7050
7051	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7052	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7053	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7054
7055	#ifdef VBOX_STRICT
7056	VMMRZCallRing3Disable(pVCpu);
7057	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7058	# ifdef DEBUG_bird
7059	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7060	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7061	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7062	# endif
7063	NOREF(uAttr);
7064	VMMRZCallRing3Enable(pVCpu);
7065	#else
7066	RT_NOREF2(pVCpu, idxSel);
7067	#endif
7068	}
7069
7070
7071	/**
7072	* Imports a guest segment register from the current VMCS into the guest-CPU
7073	* context.
7074	*
7075	* @returns VBox status code.
7076	* @param pVCpu The cross context virtual CPU structure.
7077	* @param iSegReg The segment register number (X86_SREG_XXX).
7078	*
7079	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7080	* do not log!
7081	*/
7082	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7083	{
7084	Assert(iSegReg < X86_SREG_COUNT);
7085
7086	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7087	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7088	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7089	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
7090	uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
7091	#else
7092	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7093	#endif
7094	uint64_t u64Base;
7095	uint32_t u32Sel, u32Limit, u32Attr;
7096	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7097	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7098	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7099	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
7100	if (RT_SUCCESS(rc))
7101	{
7102	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7103	pSelReg->Sel = u32Sel;
7104	pSelReg->ValidSel = u32Sel;
7105	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7106	pSelReg->u32Limit = u32Limit;
7107	pSelReg->u64Base = u64Base;
7108	pSelReg->Attr.u = u32Attr;
7109	if (u32Attr & X86DESCATTR_UNUSABLE)
7110	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7111	}
7112	return rc;
7113	}
7114
7115
7116	/**
7117	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7118	*
7119	* @returns VBox status code.
7120	* @param pVCpu The cross context virtual CPU structure.
7121	* @param pSelReg Pointer to the segment selector.
7122	*
7123	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7124	* do not log!
7125	*/
7126	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7127	{
7128	uint64_t u64Base;
7129	uint32_t u32Sel, u32Limit, u32Attr;
7130	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7131	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7132	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7133	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7134
7135	if (RT_SUCCESS(rc))
7136	{
7137	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7138	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7139	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7140	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7141	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7142	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7143	if (u32Attr & X86DESCATTR_UNUSABLE)
7144	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7145	}
7146	return rc;
7147	}
7148
7149
7150	/**
7151	* Imports the guest TR from the current VMCS into the guest-CPU context.
7152	*
7153	* @returns VBox status code.
7154	* @param pVCpu The cross context virtual CPU structure.
7155	* @param pSelReg Pointer to the segment selector.
7156	*
7157	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7158	* do not log!
7159	*/
7160	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7161	{
7162	uint32_t u32Sel, u32Limit, u32Attr;
7163	uint64_t u64Base;
7164	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7165	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7166	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7167	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7168	AssertRCReturn(rc, rc);
7169
7170	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7171	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7172	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7173	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7174	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7175	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7176	/* TR is the only selector that can never be unusable. */
7177	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7178	return VINF_SUCCESS;
7179	}
7180
7181
7182	/**
7183	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7184	*
7185	* @returns VBox status code.
7186	* @param pVCpu The cross context virtual CPU structure.
7187	*
7188	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7189	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7190	* instead!!!
7191	*/
7192	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7193	{
7194	uint64_t u64Val;
7195	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7196	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7197	{
7198	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7199	if (RT_SUCCESS(rc))
7200	{
7201	pCtx->rip = u64Val;
7202	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7203	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7204	}
7205	return rc;
7206	}
7207	return VINF_SUCCESS;
7208	}
7209
7210
7211	/**
7212	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7213	*
7214	* @returns VBox status code.
7215	* @param pVCpu The cross context virtual CPU structure.
7216	* @param pVmcsInfo The VMCS info. object.
7217	*
7218	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7219	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7220	* instead!!!
7221	*/
7222	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7223	{
7224	uint32_t u32Val;
7225	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7226	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7227	{
7228	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7229	if (RT_SUCCESS(rc))
7230	{
7231	pCtx->eflags.u32 = u32Val;
7232
7233	/* Restore eflags for real-on-v86-mode hack. */
7234	if (pVmcsInfo->RealMode.fRealOnV86Active)
7235	{
7236	pCtx->eflags.Bits.u1VM = 0;
7237	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7238	}
7239	}
7240	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7241	return rc;
7242	}
7243	return VINF_SUCCESS;
7244	}
7245
7246
7247	/**
7248	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7249	* context.
7250	*
7251	* @returns VBox status code.
7252	* @param pVCpu The cross context virtual CPU structure.
7253	* @param pVmcsInfo The VMCS info. object.
7254	*
7255	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7256	* do not log!
7257	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7258	* instead!!!
7259	*/
7260	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7261	{
7262	uint32_t u32Val;
7263	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7264	if (RT_SUCCESS(rc))
7265	{
7266	if (!u32Val)
7267	{
7268	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7269	{
7270	rc = hmR0VmxImportGuestRip(pVCpu);
7271	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7272	AssertRCReturn(rc, rc);
7273	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7274	}
7275
7276	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7277	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7278	}
7279	else
7280	{
7281	/*
7282	* We must import RIP here to set our EM interrupt-inhibited state.
7283	* We also import RFLAGS as our code that evaluates pending interrupts
7284	* before VM-entry requires it.
7285	*/
7286	rc = hmR0VmxImportGuestRip(pVCpu);
7287	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7288	if (RT_SUCCESS(rc))
7289	{
7290	if (u32Val & ( VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
7291	\| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7292	{
7293	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7294	}
7295	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7296	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7297
7298	if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
7299	{
7300	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7301	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
7302	}
7303	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7304	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7305	}
7306	}
7307	}
7308	return rc;
7309	}
7310
7311
7312	/**
7313	* Worker for VMXR0ImportStateOnDemand.
7314	*
7315	* @returns VBox status code.
7316	* @param pVCpu The cross context virtual CPU structure.
7317	* @param pVmcsInfo The VMCS info. object.
7318	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7319	*/
7320	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7321	{
7322	#define VMXLOCAL_BREAK_RC(a_rc) \
7323	if (RT_SUCCESS(a_rc)) \
7324	{ } \
7325	else \
7326	break
7327
7328	int rc = VINF_SUCCESS;
7329	PVM pVM = pVCpu->CTX_SUFF(pVM);
7330	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7331	uint64_t u64Val;
7332	uint32_t u32Val;
7333
7334	STAM_PROFILE_ADV_START(& pVCpu->hm.s.StatImportGuestState, x);
7335
7336	/*
7337	* We disable interrupts to make the updating of the state and in particular
7338	* the fExtrn modification atomic wrt to preemption hooks.
7339	*/
7340	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7341
7342	fWhat &= pCtx->fExtrn;
7343	if (fWhat)
7344	{
7345	do
7346	{
7347	if (fWhat & CPUMCTX_EXTRN_RIP)
7348	{
7349	rc = hmR0VmxImportGuestRip(pVCpu);
7350	VMXLOCAL_BREAK_RC(rc);
7351	}
7352
7353	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7354	{
7355	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7356	VMXLOCAL_BREAK_RC(rc);
7357	}
7358
7359	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7360	{
7361	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7362	VMXLOCAL_BREAK_RC(rc);
7363	}
7364
7365	if (fWhat & CPUMCTX_EXTRN_RSP)
7366	{
7367	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
7368	VMXLOCAL_BREAK_RC(rc);
7369	pCtx->rsp = u64Val;
7370	}
7371
7372	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7373	{
7374	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7375	if (fWhat & CPUMCTX_EXTRN_CS)
7376	{
7377	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7378	rc \|= hmR0VmxImportGuestRip(pVCpu);
7379	if (fRealOnV86Active)
7380	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7381	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7382	}
7383	if (fWhat & CPUMCTX_EXTRN_SS)
7384	{
7385	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7386	if (fRealOnV86Active)
7387	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7388	}
7389	if (fWhat & CPUMCTX_EXTRN_DS)
7390	{
7391	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7392	if (fRealOnV86Active)
7393	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7394	}
7395	if (fWhat & CPUMCTX_EXTRN_ES)
7396	{
7397	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7398	if (fRealOnV86Active)
7399	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7400	}
7401	if (fWhat & CPUMCTX_EXTRN_FS)
7402	{
7403	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7404	if (fRealOnV86Active)
7405	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7406	}
7407	if (fWhat & CPUMCTX_EXTRN_GS)
7408	{
7409	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7410	if (fRealOnV86Active)
7411	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7412	}
7413	VMXLOCAL_BREAK_RC(rc);
7414	}
7415
7416	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7417	{
7418	if (fWhat & CPUMCTX_EXTRN_LDTR)
7419	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
7420
7421	if (fWhat & CPUMCTX_EXTRN_GDTR)
7422	{
7423	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
7424	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
7425	pCtx->gdtr.pGdt = u64Val;
7426	pCtx->gdtr.cbGdt = u32Val;
7427	}
7428
7429	/* Guest IDTR. */
7430	if (fWhat & CPUMCTX_EXTRN_IDTR)
7431	{
7432	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
7433	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
7434	pCtx->idtr.pIdt = u64Val;
7435	pCtx->idtr.cbIdt = u32Val;
7436	}
7437
7438	/* Guest TR. */
7439	if (fWhat & CPUMCTX_EXTRN_TR)
7440	{
7441	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7442	don't need to import that one. */
7443	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7444	rc \|= hmR0VmxImportGuestTr(pVCpu);
7445	}
7446	VMXLOCAL_BREAK_RC(rc);
7447	}
7448
7449	if (fWhat & CPUMCTX_EXTRN_DR7)
7450	{
7451	if (!pVCpu->hm.s.fUsingHyperDR7)
7452	{
7453	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
7454	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
7455	VMXLOCAL_BREAK_RC(rc);
7456	pCtx->dr[7] = u32Val;
7457	}
7458	}
7459
7460	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7461	{
7462	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
7463	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
7464	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
7465	pCtx->SysEnter.cs = u32Val;
7466	VMXLOCAL_BREAK_RC(rc);
7467	}
7468
7469	#if HC_ARCH_BITS == 64
7470	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7471	{
7472	if ( pVM->hm.s.fAllow64BitGuests
7473	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7474	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7475	}
7476
7477	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7478	{
7479	if ( pVM->hm.s.fAllow64BitGuests
7480	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7481	{
7482	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7483	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7484	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7485	}
7486	}
7487	#endif
7488
7489	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7490	#if HC_ARCH_BITS == 32
7491	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
7492	#endif
7493	)
7494	{
7495	PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7496	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7497	Assert(cMsrs == 0 \|\| pMsr != NULL);
7498	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
7499	{
7500	switch (pMsr->u32Msr)
7501	{
7502	#if HC_ARCH_BITS == 32
7503	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
7504	case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
7505	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
7506	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
7507	#endif
7508	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
7509	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
7510	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7511
7512	default:
7513	{
7514	pVCpu->hm.s.u32HMError = pMsr->u32Msr;
7515	ASMSetFlags(fEFlags);
7516	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
7517	cMsrs));
7518	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7519	}
7520	}
7521	}
7522	}
7523
7524	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7525	{
7526	uint64_t u64Shadow;
7527	if (fWhat & CPUMCTX_EXTRN_CR0)
7528	{
7529	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7530	* remove when we drop 32-bit host w/ 64-bit host support, see
7531	* @bugref{9180#c39}. */
7532	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7533	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
7534	VMXLOCAL_BREAK_RC(rc);
7535	u64Val = u32Val;
7536	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7537	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7538	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7539	CPUMSetGuestCR0(pVCpu, u64Val);
7540	VMMRZCallRing3Enable(pVCpu);
7541	}
7542
7543	if (fWhat & CPUMCTX_EXTRN_CR4)
7544	{
7545	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7546	* remove when we drop 32-bit host w/ 64-bit host support, see
7547	* @bugref{9180#c39}. */
7548	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7549	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
7550	VMXLOCAL_BREAK_RC(rc);
7551	u64Val = u32Val;
7552	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
7553	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7554	pCtx->cr4 = u64Val;
7555	}
7556
7557	if (fWhat & CPUMCTX_EXTRN_CR3)
7558	{
7559	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7560	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7561	\|\| ( pVM->hm.s.fNestedPaging
7562	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7563	{
7564	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
7565	if (pCtx->cr3 != u64Val)
7566	{
7567	pCtx->cr3 = u64Val;
7568	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7569	}
7570
7571	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7572	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7573	if (CPUMIsGuestInPAEModeEx(pCtx))
7574	{
7575	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
7576	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
7577	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
7578	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
7579	VMXLOCAL_BREAK_RC(rc);
7580	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7581	}
7582	}
7583	}
7584	}
7585	} while (0);
7586
7587	if (RT_SUCCESS(rc))
7588	{
7589	/* Update fExtrn. */
7590	pCtx->fExtrn &= ~fWhat;
7591
7592	/* If everything has been imported, clear the HM keeper bit. */
7593	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7594	{
7595	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7596	Assert(!pCtx->fExtrn);
7597	}
7598	}
7599	}
7600	else
7601	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7602
7603	ASMSetFlags(fEFlags);
7604
7605	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7606
7607	if (RT_SUCCESS(rc))
7608	{ /* likely */ }
7609	else
7610	return rc;
7611
7612	/*
7613	* Honor any pending CR3 updates.
7614	*
7615	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
7616	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7617	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7618	*
7619	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7620	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7621	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7622	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7623	*
7624	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7625	*/
7626	if (VMMRZCallRing3IsEnabled(pVCpu))
7627	{
7628	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7629	{
7630	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7631	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7632	}
7633
7634	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7635	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7636
7637	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7638	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7639	}
7640
7641	return VINF_SUCCESS;
7642	#undef VMXLOCAL_BREAK_RC
7643	}
7644
7645
7646	/**
7647	* Saves the guest state from the VMCS into the guest-CPU context.
7648	*
7649	* @returns VBox status code.
7650	* @param pVCpu The cross context virtual CPU structure.
7651	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7652	*/
7653	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
7654	{
7655	PCVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7656	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7657	}
7658
7659
7660	/**
7661	* Check per-VM and per-VCPU force flag actions that require us to go back to
7662	* ring-3 for one reason or another.
7663	*
7664	* @returns Strict VBox status code (i.e. informational status codes too)
7665	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7666	* ring-3.
7667	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7668	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7669	* interrupts)
7670	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7671	* all EMTs to be in ring-3.
7672	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7673	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7674	* to the EM loop.
7675	*
7676	* @param pVCpu The cross context virtual CPU structure.
7677	* @param fStepping Whether we are single-stepping the guest using the
7678	* hypervisor debugger.
7679	*/
7680	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
7681	{
7682	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7683
7684	/*
7685	* Update pending interrupts into the APIC's IRR.
7686	*/
7687	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7688	APICUpdatePendingInterrupts(pVCpu);
7689
7690	/*
7691	* Anything pending? Should be more likely than not if we're doing a good job.
7692	*/
7693	PVM pVM = pVCpu->CTX_SUFF(pVM);
7694	if ( !fStepping
7695	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7696	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7697	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7698	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7699	return VINF_SUCCESS;
7700
7701	/* Pending PGM C3 sync. */
7702	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7703	{
7704	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7705	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7706	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7707	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7708	if (rcStrict2 != VINF_SUCCESS)
7709	{
7710	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7711	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7712	return rcStrict2;
7713	}
7714	}
7715
7716	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7717	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7718	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7719	{
7720	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7721	int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7722	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7723	return rc2;
7724	}
7725
7726	/* Pending VM request packets, such as hardware interrupts. */
7727	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7728	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7729	{
7730	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7731	return VINF_EM_PENDING_REQUEST;
7732	}
7733
7734	/* Pending PGM pool flushes. */
7735	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7736	{
7737	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7738	return VINF_PGM_POOL_FLUSH_PENDING;
7739	}
7740
7741	/* Pending DMA requests. */
7742	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7743	{
7744	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7745	return VINF_EM_RAW_TO_R3;
7746	}
7747
7748	return VINF_SUCCESS;
7749	}
7750
7751
7752	/**
7753	* Converts any TRPM trap into a pending HM event. This is typically used when
7754	* entering from ring-3 (not longjmp returns).
7755	*
7756	* @param pVCpu The cross context virtual CPU structure.
7757	*/
7758	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7759	{
7760	Assert(TRPMHasTrap(pVCpu));
7761	Assert(!pVCpu->hm.s.Event.fPending);
7762
7763	uint8_t uVector;
7764	TRPMEVENT enmTrpmEvent;
7765	RTGCUINT uErrCode;
7766	RTGCUINTPTR GCPtrFaultAddress;
7767	uint8_t cbInstr;
7768
7769	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7770	AssertRC(rc);
7771
7772	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7773	uint32_t u32IntInfo = uVector \| VMX_EXIT_INT_INFO_VALID;
7774	if (enmTrpmEvent == TRPM_TRAP)
7775	{
7776	/** @todo r=ramshankar: TRPM currently offers no way to determine a #DB that was
7777	* generated using INT1 (ICEBP). */
7778	switch (uVector)
7779	{
7780	case X86_XCPT_NMI:
7781	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7782	break;
7783
7784	case X86_XCPT_BP:
7785	case X86_XCPT_OF:
7786	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7787	break;
7788
7789	case X86_XCPT_PF:
7790	case X86_XCPT_DF:
7791	case X86_XCPT_TS:
7792	case X86_XCPT_NP:
7793	case X86_XCPT_SS:
7794	case X86_XCPT_GP:
7795	case X86_XCPT_AC:
7796	u32IntInfo \|= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7797	RT_FALL_THRU();
7798	default:
7799	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7800	break;
7801	}
7802	}
7803	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7804	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7805	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7806	{
7807	switch (uVector)
7808	{
7809	case X86_XCPT_BP:
7810	case X86_XCPT_OF:
7811	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7812	break;
7813
7814	default:
7815	Assert(uVector == X86_XCPT_DB);
7816	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7817	break;
7818	}
7819	}
7820	else
7821	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7822
7823	rc = TRPMResetTrap(pVCpu);
7824	AssertRC(rc);
7825	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7826	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7827
7828	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7829	}
7830
7831
7832	/**
7833	* Converts the pending HM event into a TRPM trap.
7834	*
7835	* @param pVCpu The cross context virtual CPU structure.
7836	*/
7837	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7838	{
7839	Assert(pVCpu->hm.s.Event.fPending);
7840
7841	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7842	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7843	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
7844	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7845
7846	/* If a trap was already pending, we did something wrong! */
7847	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7848
7849	/** @todo Use HMVmxEventToTrpmEventType() later. */
7850	TRPMEVENT enmTrapType;
7851	switch (uVectorType)
7852	{
7853	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7854	enmTrapType = TRPM_HARDWARE_INT;
7855	break;
7856
7857	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7858	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7859	enmTrapType = TRPM_TRAP;
7860	break;
7861
7862	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT: /* #DB (INT1/ICEBP). */
7863	Assert(uVector == X86_XCPT_DB);
7864	enmTrapType = TRPM_SOFTWARE_INT;
7865	break;
7866
7867	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP (INT3) and #OF (INTO) */
7868	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7869	enmTrapType = TRPM_SOFTWARE_INT;
7870	break;
7871
7872	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7873	enmTrapType = TRPM_SOFTWARE_INT;
7874	break;
7875
7876	default:
7877	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7878	enmTrapType = TRPM_32BIT_HACK;
7879	break;
7880	}
7881
7882	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7883
7884	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7885	AssertRC(rc);
7886
7887	if (fErrorCodeValid)
7888	TRPMSetErrorCode(pVCpu, uErrorCode);
7889
7890	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7891	&& uVector == X86_XCPT_PF)
7892	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7893	else if (enmTrapType == TRPM_SOFTWARE_INT)
7894	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7895
7896	/* We're now done converting the pending event. */
7897	pVCpu->hm.s.Event.fPending = false;
7898	}
7899
7900
7901	/**
7902	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7903	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7904	*
7905	* @param pVCpu The cross context virtual CPU structure.
7906	* @param pVmcsInfo The VMCS info. object.
7907	*/
7908	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
7909	{
7910	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7911	{
7912	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7913	{
7914	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7915	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7916	AssertRC(rc);
7917	}
7918	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
7919	}
7920
7921
7922	/**
7923	* Clears the interrupt-window exiting control in the VMCS.
7924	*
7925	* @param pVmcsInfo The VMCS info. object.
7926	*/
7927	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7928	{
7929	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7930	{
7931	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7932	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7933	}
7934	return VINF_SUCCESS;
7935	}
7936
7937
7938	/**
7939	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7940	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7941	*
7942	* @param pVCpu The cross context virtual CPU structure.
7943	* @param pVmcsInfo The VMCS info. object.
7944	*/
7945	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
7946	{
7947	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7948	{
7949	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7950	{
7951	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7952	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7953	AssertRC(rc);
7954	Log4Func(("Setup NMI-window exiting\n"));
7955	}
7956	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7957	}
7958
7959
7960	/**
7961	* Clears the NMI-window exiting control in the VMCS.
7962	*
7963	* @param pVmcsInfo The VMCS info. object.
7964	*/
7965	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7966	{
7967	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7968	{
7969	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7970	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7971	}
7972	return VINF_SUCCESS;
7973	}
7974
7975
7976	/**
7977	* Does the necessary state syncing before returning to ring-3 for any reason
7978	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7979	*
7980	* @returns VBox status code.
7981	* @param pVCpu The cross context virtual CPU structure.
7982	* @param fImportState Whether to import the guest state from the VMCS back
7983	* to the guest-CPU context.
7984	*
7985	* @remarks No-long-jmp zone!!!
7986	*/
7987	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
7988	{
7989	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7990	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7991
7992	RTCPUID idCpu = RTMpCpuId();
7993	Log4Func(("HostCpuId=%u\n", idCpu));
7994
7995	/*
7996	* !!! IMPORTANT !!!
7997	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7998	*/
7999
8000	/* Save the guest state if necessary. */
8001	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8002	if (fImportState)
8003	{
8004	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8005	AssertRCReturn(rc, rc);
8006	}
8007
8008	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8009	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8010	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8011
8012	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8013	#ifdef VBOX_STRICT
8014	if (CPUMIsHyperDebugStateActive(pVCpu))
8015	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8016	#endif
8017	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8018	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8019	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8020
8021	#if HC_ARCH_BITS == 64
8022	/* Restore host-state bits that VT-x only restores partially. */
8023	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8024	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8025	{
8026	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8027	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8028	}
8029	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8030	#endif
8031
8032	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8033	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8034	{
8035	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8036	if (!fImportState)
8037	{
8038	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8039	AssertRCReturn(rc, rc);
8040	}
8041	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8042	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8043	}
8044	else
8045	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8046
8047	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8048	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8049
8050	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8051	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8052	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8053	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8054	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8055	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8056	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8057	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8058	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8059
8060	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8061
8062	/** @todo This partially defeats the purpose of having preemption hooks.
8063	* The problem is, deregistering the hooks should be moved to a place that
8064	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8065	* context.
8066	*/
8067	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8068	AssertRCReturn(rc, rc);
8069
8070	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8071	NOREF(idCpu);
8072	return VINF_SUCCESS;
8073	}
8074
8075
8076	/**
8077	* Leaves the VT-x session.
8078	*
8079	* @returns VBox status code.
8080	* @param pVCpu The cross context virtual CPU structure.
8081	*
8082	* @remarks No-long-jmp zone!!!
8083	*/
8084	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8085	{
8086	HM_DISABLE_PREEMPT(pVCpu);
8087	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8088	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8089	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8090
8091	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8092	and done this from the VMXR0ThreadCtxCallback(). */
8093	if (!pVCpu->hm.s.fLeaveDone)
8094	{
8095	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8096	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8097	pVCpu->hm.s.fLeaveDone = true;
8098	}
8099	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8100
8101	/*
8102	* !!! IMPORTANT !!!
8103	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8104	*/
8105
8106	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8107	/** @todo Deregistering here means we need to VMCLEAR always
8108	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8109	* for calling VMMR0ThreadCtxHookDisable here! */
8110	VMMR0ThreadCtxHookDisable(pVCpu);
8111
8112	/* Leave HM context. This takes care of local init (term). */
8113	int rc = HMR0LeaveCpu(pVCpu);
8114
8115	HM_RESTORE_PREEMPT();
8116	return rc;
8117	}
8118
8119
8120	/**
8121	* Does the necessary state syncing before doing a longjmp to ring-3.
8122	*
8123	* @returns VBox status code.
8124	* @param pVCpu The cross context virtual CPU structure.
8125	*
8126	* @remarks No-long-jmp zone!!!
8127	*/
8128	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8129	{
8130	return hmR0VmxLeaveSession(pVCpu);
8131	}
8132
8133
8134	/**
8135	* Take necessary actions before going back to ring-3.
8136	*
8137	* An action requires us to go back to ring-3. This function does the necessary
8138	* steps before we can safely return to ring-3. This is not the same as longjmps
8139	* to ring-3, this is voluntary and prepares the guest so it may continue
8140	* executing outside HM (recompiler/IEM).
8141	*
8142	* @returns VBox status code.
8143	* @param pVCpu The cross context virtual CPU structure.
8144	* @param rcExit The reason for exiting to ring-3. Can be
8145	* VINF_VMM_UNKNOWN_RING3_CALL.
8146	*/
8147	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8148	{
8149	Assert(pVCpu);
8150	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8151
8152	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8153	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8154	{
8155	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8156	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8157	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8158	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8159	}
8160
8161	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8162	VMMRZCallRing3Disable(pVCpu);
8163	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8164
8165	/*
8166	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8167	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8168	*
8169	* This is because execution may continue from ring-3 and we would need to inject
8170	* the event from there (hence place it back in TRPM).
8171	*/
8172	if (pVCpu->hm.s.Event.fPending)
8173	{
8174	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8175	Assert(!pVCpu->hm.s.Event.fPending);
8176
8177	/* Clear the events from the VMCS. */
8178	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8179	AssertRCReturn(rc, rc);
8180	}
8181	#ifdef VBOX_STRICT
8182	else
8183	{
8184	/*
8185	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8186	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8187	* occasionally, see @bugref{9180#c42}.
8188	*/
8189	uint32_t uEntryIntInfo;
8190	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8191	AssertRC(rc);
8192	Assert(!VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8193	}
8194	#endif
8195
8196	/*
8197	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8198	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8199	* (e.g. TPR below threshold).
8200	*/
8201	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8202	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8203	AssertRCReturn(rc, rc);
8204
8205	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8206	and if we're injecting an event we should have a TRPM trap pending. */
8207	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8208	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8209	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8210	#endif
8211
8212	/* Save guest state and restore host state bits. */
8213	rc = hmR0VmxLeaveSession(pVCpu);
8214	AssertRCReturn(rc, rc);
8215	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8216
8217	/* Thread-context hooks are unregistered at this point!!! */
8218
8219	/* Sync recompiler state. */
8220	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8221	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8222	\| CPUM_CHANGED_LDTR
8223	\| CPUM_CHANGED_GDTR
8224	\| CPUM_CHANGED_IDTR
8225	\| CPUM_CHANGED_TR
8226	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8227	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8228	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8229	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8230
8231	Assert(!pVCpu->hm.s.fClearTrapFlag);
8232
8233	/* Update the exit-to-ring 3 reason. */
8234	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8235
8236	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8237	if ( rcExit != VINF_EM_RAW_INTERRUPT
8238	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8239	{
8240	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8241	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8242	}
8243
8244	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8245
8246	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
8247	VMMRZCallRing3RemoveNotification(pVCpu);
8248	VMMRZCallRing3Enable(pVCpu);
8249
8250	return rc;
8251	}
8252
8253
8254	/**
8255	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8256	* longjump to ring-3 and possibly get preempted.
8257	*
8258	* @returns VBox status code.
8259	* @param pVCpu The cross context virtual CPU structure.
8260	* @param enmOperation The operation causing the ring-3 longjump.
8261	* @param pvUser User argument, currently unused, NULL.
8262	*/
8263	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
8264	{
8265	RT_NOREF(pvUser);
8266	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8267	{
8268	/*
8269	* !!! IMPORTANT !!!
8270	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8271	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8272	*/
8273	VMMRZCallRing3RemoveNotification(pVCpu);
8274	VMMRZCallRing3Disable(pVCpu);
8275	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
8276	RTThreadPreemptDisable(&PreemptState);
8277
8278	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8279	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8280	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8281	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8282
8283	#if HC_ARCH_BITS == 64
8284	/* Restore host-state bits that VT-x only restores partially. */
8285	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8286	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8287	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8288	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8289	#endif
8290
8291	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8292	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8293	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8294
8295	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8296	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8297	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8298
8299	/* Clear the current VMCS data back to memory. */
8300	hmR0VmxClearVmcs(pVmcsInfo);
8301
8302	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8303	VMMR0ThreadCtxHookDisable(pVCpu);
8304	HMR0LeaveCpu(pVCpu);
8305	RTThreadPreemptRestore(&PreemptState);
8306	return VINF_SUCCESS;
8307	}
8308
8309	Assert(pVCpu);
8310	Assert(pvUser);
8311	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8312	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8313
8314	VMMRZCallRing3Disable(pVCpu);
8315	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8316
8317	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8318
8319	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8320	AssertRCReturn(rc, rc);
8321
8322	VMMRZCallRing3Enable(pVCpu);
8323	return VINF_SUCCESS;
8324	}
8325
8326
8327	/**
8328	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8329	* stack.
8330	*
8331	* @returns Strict VBox status code (i.e. informational status codes too).
8332	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8333	* @param pVCpu The cross context virtual CPU structure.
8334	* @param uValue The value to push to the guest stack.
8335	*/
8336	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
8337	{
8338	/*
8339	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8340	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8341	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8342	*/
8343	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8344	if (pCtx->sp == 1)
8345	return VINF_EM_RESET;
8346	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8347	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8348	AssertRC(rc);
8349	return rc;
8350	}
8351
8352
8353	/**
8354	* Injects an event into the guest upon VM-entry by updating the relevant fields
8355	* in the VM-entry area in the VMCS.
8356	*
8357	* @returns Strict VBox status code (i.e. informational status codes too).
8358	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8359	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8360	*
8361	* @param pVCpu The cross context virtual CPU structure.
8362	* @param pVmxTransient The VMX-transient structure.
8363	* @param pEvent The event being injected.
8364	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state.
8365	* This will be updated if necessary. This cannot not
8366	* be NULL.
8367	* @param fStepping Whether we're single-stepping guest execution and
8368	* should return VINF_EM_DBG_STEPPED if the event is
8369	* injected directly (registers modified by us, not by
8370	* hardware on VM-entry).
8371	*/
8372	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8373	uint32_t *pfIntrState)
8374	{
8375	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8376	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8377	Assert(pfIntrState);
8378
8379	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8380	uint32_t u32IntInfo = pEvent->u64IntInfo;
8381	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8382	uint32_t const cbInstr = pEvent->cbInstr;
8383	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8384	uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8385	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8386
8387	#ifdef VBOX_STRICT
8388	/*
8389	* Validate the error-code-valid bit for hardware exceptions.
8390	* No error codes for exceptions in real-mode.
8391	*
8392	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8393	*/
8394	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8395	&& !CPUMIsGuestInRealModeEx(pCtx))
8396	{
8397	switch (uVector)
8398	{
8399	case X86_XCPT_PF:
8400	case X86_XCPT_DF:
8401	case X86_XCPT_TS:
8402	case X86_XCPT_NP:
8403	case X86_XCPT_SS:
8404	case X86_XCPT_GP:
8405	case X86_XCPT_AC:
8406	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8407	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8408	RT_FALL_THRU();
8409	default:
8410	break;
8411	}
8412	}
8413
8414	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8415	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8416	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8417	#endif
8418
8419	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8420
8421	/*
8422	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8423	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8424	* interrupt handler in the (real-mode) guest.
8425	*
8426	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8427	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8428	*/
8429	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8430	{
8431	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8432	{
8433	/*
8434	* For CPUs with unrestricted guest execution enabled and with the guest
8435	* in real-mode, we must not set the deliver-error-code bit.
8436	*
8437	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8438	*/
8439	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8440	}
8441	else
8442	{
8443	PVM pVM = pVCpu->CTX_SUFF(pVM);
8444	Assert(PDMVmmDevHeapIsEnabled(pVM));
8445	Assert(pVM->hm.s.vmx.pRealModeTSS);
8446	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8447
8448	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8449	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8450	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8451	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8452	AssertRCReturn(rc2, rc2);
8453
8454	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8455	size_t const cbIdtEntry = sizeof(X86IDTR16);
8456	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8457	{
8458	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8459	if (uVector == X86_XCPT_DF)
8460	return VINF_EM_RESET;
8461
8462	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8463	No error codes for exceptions in real-mode. */
8464	if (uVector == X86_XCPT_GP)
8465	{
8466	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8467	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8468	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8469	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8470	HMEVENT EventXcptDf;
8471	RT_ZERO(EventXcptDf);
8472	EventXcptDf.u64IntInfo = uXcptDfInfo;
8473	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8474	}
8475
8476	/*
8477	* If we're injecting an event with no valid IDT entry, inject a #GP.
8478	* No error codes for exceptions in real-mode.
8479	*
8480	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8481	*/
8482	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8483	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8484	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8485	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8486	HMEVENT EventXcptGp;
8487	RT_ZERO(EventXcptGp);
8488	EventXcptGp.u64IntInfo = uXcptGpInfo;
8489	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8490	}
8491
8492	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8493	uint16_t uGuestIp = pCtx->ip;
8494	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8495	{
8496	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8497	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8498	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8499	}
8500	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8501	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8502
8503	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8504	X86IDTR16 IdtEntry;
8505	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8506	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8507	AssertRCReturn(rc2, rc2);
8508
8509	/* Construct the stack frame for the interrupt/exception handler. */
8510	VBOXSTRICTRC rcStrict;
8511	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8512	if (rcStrict == VINF_SUCCESS)
8513	{
8514	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8515	if (rcStrict == VINF_SUCCESS)
8516	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8517	}
8518
8519	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8520	if (rcStrict == VINF_SUCCESS)
8521	{
8522	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8523	pCtx->rip = IdtEntry.offSel;
8524	pCtx->cs.Sel = IdtEntry.uSel;
8525	pCtx->cs.ValidSel = IdtEntry.uSel;
8526	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8527	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8528	&& uVector == X86_XCPT_PF)
8529	pCtx->cr2 = GCPtrFault;
8530
8531	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8532	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8533	\| HM_CHANGED_GUEST_RSP);
8534
8535	/*
8536	* If we delivered a hardware exception (other than an NMI) and if there was
8537	* block-by-STI in effect, we should clear it.
8538	*/
8539	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8540	{
8541	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8542	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8543	Log4Func(("Clearing inhibition due to STI\n"));
8544	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8545	}
8546
8547	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8548	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8549
8550	/*
8551	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8552	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8553	*/
8554	pVCpu->hm.s.Event.fPending = false;
8555
8556	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8557	if (fStepping)
8558	rcStrict = VINF_EM_DBG_STEPPED;
8559	}
8560	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8561	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8562	return rcStrict;
8563	}
8564	}
8565
8566	/*
8567	* Validate.
8568	*/
8569	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8570	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8571
8572	/*
8573	* Inject the event into the VMCS.
8574	*/
8575	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8576	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8577	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8578	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8579	AssertRCReturn(rc, rc);
8580
8581	/*
8582	* Update guest CR2 if this is a page-fault.
8583	*/
8584	if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8585	&& uVector == X86_XCPT_PF)
8586	pCtx->cr2 = GCPtrFault;
8587
8588	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8589	return VINF_SUCCESS;
8590	}
8591
8592
8593	/**
8594	* Evaluates the event to be delivered to the guest and sets it as the pending
8595	* event.
8596	*
8597	* @returns Strict VBox status code (i.e. informational status codes too).
8598	* @param pVCpu The cross context virtual CPU structure.
8599	* @param pVmcsInfo The VMCS info. object.
8600	* @param pVmxTransient The VMX-transient structure.
8601	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8602	*/
8603	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8604	{
8605	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8606	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8607
8608	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
8609	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmcsInfo);
8610	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8611	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8612	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8613
8614	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8615	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8616	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8617	Assert(!TRPMHasTrap(pVCpu));
8618	Assert(pfIntrState);
8619
8620	*pfIntrState = fIntrState;
8621
8622	/*
8623	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
8624	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
8625	*/
8626	/** @todo SMI. SMIs take priority over NMIs. */
8627	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
8628	{
8629	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8630	if ( !pVCpu->hm.s.Event.fPending
8631	&& !fBlockNmi
8632	&& !fBlockSti
8633	&& !fBlockMovSS)
8634	{
8635	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8636	if ( pVmxTransient->fIsNestedGuest
8637	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8638	return IEMExecVmxVmexitNmi(pVCpu);
8639	#endif
8640	hmR0VmxSetPendingXcptNmi(pVCpu);
8641	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8642	Log4Func(("Pending NMI\n"));
8643	}
8644	else
8645	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8646	}
8647	/*
8648	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
8649	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
8650	*/
8651	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8652	&& !pVCpu->hm.s.fSingleInstruction)
8653	{
8654	Assert(!DBGFIsStepping(pVCpu));
8655	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8656	AssertRCReturn(rc, rc);
8657	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8658	if ( !pVCpu->hm.s.Event.fPending
8659	&& !fBlockInt
8660	&& !fBlockSti
8661	&& !fBlockMovSS)
8662	{
8663	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8664	if ( pVmxTransient->fIsNestedGuest
8665	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
8666	{
8667	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector /, true / fIntPending */);
8668	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8669	return rcStrict;
8670	}
8671	#endif
8672	uint8_t u8Interrupt;
8673	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8674	if (RT_SUCCESS(rc))
8675	{
8676	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8677	if ( pVmxTransient->fIsNestedGuest
8678	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8679	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8680	{
8681	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8682	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8683	return rcStrict;
8684	}
8685	#endif
8686	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8687	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
8688	}
8689	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8690	{
8691	if ( !pVmxTransient->fIsNestedGuest
8692	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8693	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
8694	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8695
8696	/*
8697	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8698	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8699	* need to re-set this force-flag here.
8700	*/
8701	}
8702	else
8703	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8704	}
8705	else
8706	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8707	}
8708
8709	return VINF_SUCCESS;
8710	}
8711
8712
8713	/**
8714	* Injects any pending events into the guest if the guest is in a state to
8715	* receive them.
8716	*
8717	* @returns Strict VBox status code (i.e. informational status codes too).
8718	* @param pVCpu The cross context virtual CPU structure.
8719	* @param pVmxTransient The VMX-transient structure.
8720	* @param fIntrState The VT-x guest-interruptibility state.
8721	* @param fStepping Whether we are single-stepping the guest using the
8722	* hypervisor debugger and should return
8723	* VINF_EM_DBG_STEPPED if the event was dispatched
8724	* directly.
8725	*/
8726	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
8727	{
8728	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8729	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8730
8731	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8732	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8733
8734	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8735	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8736	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8737	Assert(!TRPMHasTrap(pVCpu));
8738
8739	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8740	if (pVCpu->hm.s.Event.fPending)
8741	{
8742	/*
8743	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8744	* pending even while injecting an event and in this case, we want a VM-exit as soon as
8745	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
8746	*
8747	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8748	*/
8749	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8750	#ifdef VBOX_STRICT
8751	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8752	{
8753	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
8754	Assert(!fBlockInt);
8755	Assert(!fBlockSti);
8756	Assert(!fBlockMovSS);
8757	}
8758	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
8759	{
8760	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8761	Assert(!fBlockSti);
8762	Assert(!fBlockMovSS);
8763	Assert(!fBlockNmi);
8764	}
8765	#endif
8766	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8767	uIntType));
8768
8769	/*
8770	* Inject the event and get any changes to the guest-interruptibility state.
8771	*
8772	* The guest-interruptibility state may need to be updated if we inject the event
8773	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
8774	*/
8775	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
8776	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8777
8778	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8779	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8780	else
8781	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8782	}
8783
8784	/*
8785	* Update the guest-interruptibility state.
8786	*
8787	* This is required for the real-on-v86 software interrupt injection case above, as well as
8788	* updates to the guest state from ring-3 or IEM/REM.
8789	*/
8790	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
8791	AssertRCReturn(rc, rc);
8792
8793	/*
8794	* There's no need to clear the VM-entry interruption-information field here if we're not
8795	* injecting anything. VT-x clears the valid bit on every VM-exit.
8796	*
8797	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8798	*/
8799
8800	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8801	NOREF(fBlockMovSS); NOREF(fBlockSti);
8802	return rcStrict;
8803	}
8804
8805
8806	/**
8807	* Enters the VT-x session.
8808	*
8809	* @returns VBox status code.
8810	* @param pVCpu The cross context virtual CPU structure.
8811	*/
8812	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
8813	{
8814	AssertPtr(pVCpu);
8815	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8816	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8817
8818	LogFlowFunc(("pVCpu=%p\n", pVCpu));
8819	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8820	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8821
8822	#ifdef VBOX_STRICT
8823	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8824	RTCCUINTREG uHostCR4 = ASMGetCR4();
8825	if (!(uHostCR4 & X86_CR4_VMXE))
8826	{
8827	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8828	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8829	}
8830	#endif
8831
8832	/*
8833	* Load the appropriate VMCS as the current and active one.
8834	*/
8835	PVMXVMCSINFO pVmcsInfo;
8836	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
8837	if (!fInNestedGuestMode)
8838	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
8839	else
8840	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
8841	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
8842	if (RT_SUCCESS(rc))
8843	{
8844	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
8845	pVCpu->hm.s.fLeaveDone = false;
8846	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8847
8848	/*
8849	* Do the EMT scheduled L1D flush here if needed.
8850	*/
8851	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8852	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8853	}
8854	return rc;
8855	}
8856
8857
8858	/**
8859	* The thread-context callback (only on platforms which support it).
8860	*
8861	* @param enmEvent The thread-context event.
8862	* @param pVCpu The cross context virtual CPU structure.
8863	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8864	* @thread EMT(pVCpu)
8865	*/
8866	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8867	{
8868	NOREF(fGlobalInit);
8869
8870	switch (enmEvent)
8871	{
8872	case RTTHREADCTXEVENT_OUT:
8873	{
8874	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8875	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8876	VMCPU_ASSERT_EMT(pVCpu);
8877
8878	/* No longjmps (logger flushes, locks) in this fragile context. */
8879	VMMRZCallRing3Disable(pVCpu);
8880	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8881
8882	/* Restore host-state (FPU, debug etc.) */
8883	if (!pVCpu->hm.s.fLeaveDone)
8884	{
8885	/*
8886	* Do -not- import the guest-state here as we might already be in the middle of importing
8887	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8888	*/
8889	hmR0VmxLeave(pVCpu, false /* fImportState */);
8890	pVCpu->hm.s.fLeaveDone = true;
8891	}
8892
8893	/* Leave HM context, takes care of local init (term). */
8894	int rc = HMR0LeaveCpu(pVCpu);
8895	AssertRC(rc);
8896
8897	/* Restore longjmp state. */
8898	VMMRZCallRing3Enable(pVCpu);
8899	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8900	break;
8901	}
8902
8903	case RTTHREADCTXEVENT_IN:
8904	{
8905	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8906	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8907	VMCPU_ASSERT_EMT(pVCpu);
8908
8909	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8910	VMMRZCallRing3Disable(pVCpu);
8911	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8912
8913	/* Initialize the bare minimum state required for HM. This takes care of
8914	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8915	int rc = hmR0EnterCpu(pVCpu);
8916	AssertRC(rc);
8917	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8918	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8919
8920	/* Load the active VMCS as the current one. */
8921	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8922	rc = hmR0VmxLoadVmcs(pVmcsInfo);
8923	AssertRC(rc);
8924	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8925	pVCpu->hm.s.fLeaveDone = false;
8926
8927	/* Do the EMT scheduled L1D flush if needed. */
8928	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8929	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8930
8931	/* Restore longjmp state. */
8932	VMMRZCallRing3Enable(pVCpu);
8933	break;
8934	}
8935
8936	default:
8937	break;
8938	}
8939	}
8940
8941
8942	/**
8943	* Exports the host state into the VMCS host-state area.
8944	* Sets up the VM-exit MSR-load area.
8945	*
8946	* The CPU state will be loaded from these fields on every successful VM-exit.
8947	*
8948	* @returns VBox status code.
8949	* @param pVCpu The cross context virtual CPU structure.
8950	*
8951	* @remarks No-long-jump zone!!!
8952	*/
8953	static int hmR0VmxExportHostState(PVMCPU pVCpu)
8954	{
8955	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8956
8957	int rc = VINF_SUCCESS;
8958	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8959	{
8960	rc = hmR0VmxExportHostControlRegs();
8961	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8962
8963	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8964	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8965
8966	rc = hmR0VmxExportHostMsrs(pVCpu);
8967	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8968
8969	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8970	}
8971	return rc;
8972	}
8973
8974
8975	/**
8976	* Saves the host state in the VMCS host-state.
8977	*
8978	* @returns VBox status code.
8979	* @param pVCpu The cross context virtual CPU structure.
8980	*
8981	* @remarks No-long-jump zone!!!
8982	*/
8983	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
8984	{
8985	AssertPtr(pVCpu);
8986	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8987
8988	/*
8989	* Export the host state here while entering HM context.
8990	* When thread-context hooks are used, we might get preempted and have to re-save the host
8991	* state but most of the time we won't be, so do it here before we disable interrupts.
8992	*/
8993	return hmR0VmxExportHostState(pVCpu);
8994	}
8995
8996
8997	/**
8998	* Exports the guest state into the VMCS guest-state area.
8999	*
9000	* The will typically be done before VM-entry when the guest-CPU state and the
9001	* VMCS state may potentially be out of sync.
9002	*
9003	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9004	* VM-entry controls.
9005	* Sets up the appropriate VMX non-root function to execute guest code based on
9006	* the guest CPU mode.
9007	*
9008	* @returns VBox strict status code.
9009	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9010	* without unrestricted guest execution and the VMMDev is not presently
9011	* mapped (e.g. EFI32).
9012	*
9013	* @param pVCpu The cross context virtual CPU structure.
9014	* @param pVmxTransient The VMX-transient structure.
9015	*
9016	* @remarks No-long-jump zone!!!
9017	*/
9018	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9019	{
9020	AssertPtr(pVCpu);
9021	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9022	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9023
9024	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9025
9026	/*
9027	* Determine real-on-v86 mode.
9028	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9029	*/
9030	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9031	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9032	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9033	pVmcsInfo->RealMode. fRealOnV86Active = false;
9034	else
9035	{
9036	Assert(!pVmxTransient->fIsNestedGuest);
9037	pVmcsInfo->RealMode.fRealOnV86Active = true;
9038	}
9039
9040	/*
9041	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9042	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9043	*/
9044	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9045	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9046	* be a need to evaluate this everytime since I'm pretty sure we intercept
9047	* all guest paging mode changes. */
9048	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9049	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9050
9051	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9052	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9053
9054	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9055	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9056
9057	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9058	if (rcStrict == VINF_SUCCESS)
9059	{ /* likely */ }
9060	else
9061	{
9062	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9063	return rcStrict;
9064	}
9065
9066	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9067	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9068
9069	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9070	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9071
9072	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9073	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9074
9075	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9076	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9077
9078	rc = hmR0VmxExportGuestRip(pVCpu);
9079	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9080	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9081	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9082
9083	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9084	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9085	\| HM_CHANGED_GUEST_CR2
9086	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9087	\| HM_CHANGED_GUEST_X87
9088	\| HM_CHANGED_GUEST_SSE_AVX
9089	\| HM_CHANGED_GUEST_OTHER_XSAVE
9090	\| HM_CHANGED_GUEST_XCRx
9091	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9092	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9093	\| HM_CHANGED_GUEST_TSC_AUX
9094	\| HM_CHANGED_GUEST_OTHER_MSRS
9095	\| HM_CHANGED_GUEST_HWVIRT /* More accurate PLE handling someday? */
9096	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9097
9098	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9099	return rc;
9100	}
9101
9102
9103	/**
9104	* Exports the state shared between the host and guest into the VMCS.
9105	*
9106	* @param pVCpu The cross context virtual CPU structure.
9107	* @param pVmxTransient The VMX-transient structure.
9108	*
9109	* @remarks No-long-jump zone!!!
9110	*/
9111	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9112	{
9113	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9114	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9115
9116	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9117	{
9118	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9119	AssertRC(rc);
9120	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9121
9122	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9123	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9124	{
9125	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9126	AssertRC(rc);
9127	}
9128	}
9129
9130	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9131	{
9132	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9133	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9134	}
9135
9136	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9137	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9138	}
9139
9140
9141	/**
9142	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9143	*
9144	* @returns Strict VBox status code (i.e. informational status codes too).
9145	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9146	* without unrestricted guest execution and the VMMDev is not presently
9147	* mapped (e.g. EFI32).
9148	*
9149	* @param pVCpu The cross context virtual CPU structure.
9150	* @param pVmxTransient The VMX-transient structure.
9151	*
9152	* @remarks No-long-jump zone!!!
9153	*/
9154	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9155	{
9156	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9157	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9158	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9159
9160	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9161	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9162	#endif
9163
9164	/*
9165	* For many exits it's only RIP that changes and hence try to export it first
9166	* without going through a lot of change flag checks.
9167	*/
9168	VBOXSTRICTRC rcStrict;
9169	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9170	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9171	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9172	{
9173	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9174	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9175	{ /* likely */}
9176	else
9177	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9178	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9179	}
9180	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9181	{
9182	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9183	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9184	{ /* likely */}
9185	else
9186	{
9187	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9188	VBOXSTRICTRC_VAL(rcStrict)));
9189	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9190	return rcStrict;
9191	}
9192	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9193	}
9194	else
9195	rcStrict = VINF_SUCCESS;
9196
9197	#ifdef VBOX_STRICT
9198	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9199	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9200	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9201	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9202	("fCtxChanged=%#RX64\n", fCtxChanged));
9203	#endif
9204	return rcStrict;
9205	}
9206
9207
9208	/**
9209	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9210	* and update error record fields accordingly.
9211	*
9212	* @return VMX_IGS_* return codes.
9213	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9214	* wrong with the guest state.
9215	*
9216	* @param pVCpu The cross context virtual CPU structure.
9217	* @param pVmcsInfo The VMCS info. object.
9218	*
9219	* @remarks This function assumes our cache of the VMCS controls
9220	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9221	*/
9222	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
9223	{
9224	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9225	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9226	uError = (err); \
9227	break; \
9228	} else do { } while (0)
9229
9230	int rc;
9231	PVM pVM = pVCpu->CTX_SUFF(pVM);
9232	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9233	uint32_t uError = VMX_IGS_ERROR;
9234	uint32_t u32Val;
9235	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9236
9237	do
9238	{
9239	/*
9240	* CR0.
9241	*/
9242	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9243	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9244	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
9245	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9246	if (fUnrestrictedGuest)
9247	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
9248
9249	uint32_t u32GuestCr0;
9250	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
9251	AssertRCBreak(rc);
9252	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9253	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9254	if ( !fUnrestrictedGuest
9255	&& (u32GuestCr0 & X86_CR0_PG)
9256	&& !(u32GuestCr0 & X86_CR0_PE))
9257	{
9258	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9259	}
9260
9261	/*
9262	* CR4.
9263	*/
9264	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9265	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9266
9267	uint32_t u32GuestCr4;
9268	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
9269	AssertRCBreak(rc);
9270	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9271	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9272
9273	/*
9274	* IA32_DEBUGCTL MSR.
9275	*/
9276	uint64_t u64Val;
9277	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9278	AssertRCBreak(rc);
9279	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9280	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9281	{
9282	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9283	}
9284	uint64_t u64DebugCtlMsr = u64Val;
9285
9286	#ifdef VBOX_STRICT
9287	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9288	AssertRCBreak(rc);
9289	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9290	#endif
9291	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9292
9293	/*
9294	* RIP and RFLAGS.
9295	*/
9296	uint32_t u32Eflags;
9297	#if HC_ARCH_BITS == 64
9298	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9299	AssertRCBreak(rc);
9300	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9301	if ( !fLongModeGuest
9302	\|\| !pCtx->cs.Attr.n.u1Long)
9303	{
9304	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9305	}
9306	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9307	* must be identical if the "IA-32e mode guest" VM-entry
9308	* control is 1 and CS.L is 1. No check applies if the
9309	* CPU supports 64 linear-address bits. */
9310
9311	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9312	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9313	AssertRCBreak(rc);
9314	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9315	VMX_IGS_RFLAGS_RESERVED);
9316	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9317	u32Eflags = u64Val;
9318	#else
9319	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9320	AssertRCBreak(rc);
9321	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9322	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9323	#endif
9324
9325	if ( fLongModeGuest
9326	\|\| ( fUnrestrictedGuest
9327	&& !(u32GuestCr0 & X86_CR0_PE)))
9328	{
9329	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9330	}
9331
9332	uint32_t u32EntryInfo;
9333	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9334	AssertRCBreak(rc);
9335	if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9336	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9337	{
9338	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9339	}
9340
9341	/*
9342	* 64-bit checks.
9343	*/
9344	#if HC_ARCH_BITS == 64
9345	if (fLongModeGuest)
9346	{
9347	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9348	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9349	}
9350
9351	if ( !fLongModeGuest
9352	&& (u32GuestCr4 & X86_CR4_PCIDE))
9353	{
9354	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9355	}
9356
9357	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9358	* 51:32 beyond the processor's physical-address width are 0. */
9359
9360	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9361	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9362	{
9363	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9364	}
9365
9366	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9367	AssertRCBreak(rc);
9368	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9369
9370	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9371	AssertRCBreak(rc);
9372	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9373	#endif
9374
9375	/*
9376	* PERF_GLOBAL MSR.
9377	*/
9378	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9379	{
9380	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9381	AssertRCBreak(rc);
9382	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9383	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9384	}
9385
9386	/*
9387	* PAT MSR.
9388	*/
9389	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9390	{
9391	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9392	AssertRCBreak(rc);
9393	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9394	for (unsigned i = 0; i < 8; i++)
9395	{
9396	uint8_t u8Val = (u64Val & 0xff);
9397	if ( u8Val != 0 /* UC */
9398	&& u8Val != 1 /* WC */
9399	&& u8Val != 4 /* WT */
9400	&& u8Val != 5 /* WP */
9401	&& u8Val != 6 /* WB */
9402	&& u8Val != 7 /* UC- */)
9403	{
9404	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9405	}
9406	u64Val >>= 8;
9407	}
9408	}
9409
9410	/*
9411	* EFER MSR.
9412	*/
9413	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9414	{
9415	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9416	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9417	AssertRCBreak(rc);
9418	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9419	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9420	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9421	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9422	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9423	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9424	* iemVmxVmentryCheckGuestState(). */
9425	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9426	\|\| !(u32GuestCr0 & X86_CR0_PG)
9427	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9428	VMX_IGS_EFER_LMA_LME_MISMATCH);
9429	}
9430
9431	/*
9432	* Segment registers.
9433	*/
9434	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9435	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9436	if (!(u32Eflags & X86_EFL_VM))
9437	{
9438	/* CS */
9439	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9440	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9441	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9442	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9443	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9444	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9445	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9446	/* CS cannot be loaded with NULL in protected mode. */
9447	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9448	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9449	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9450	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9451	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9452	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9453	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9454	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9455	else
9456	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9457
9458	/* SS */
9459	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9460	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9461	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9462	if ( !(pCtx->cr0 & X86_CR0_PE)
9463	\|\| pCtx->cs.Attr.n.u4Type == 3)
9464	{
9465	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9466	}
9467	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9468	{
9469	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9470	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9471	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9472	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9473	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9474	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9475	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9476	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9477	}
9478
9479	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9480	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9481	{
9482	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9483	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9484	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9485	\|\| pCtx->ds.Attr.n.u4Type > 11
9486	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9487	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9488	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9489	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9490	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9491	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9492	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9493	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9494	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9495	}
9496	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9497	{
9498	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9499	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9500	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9501	\|\| pCtx->es.Attr.n.u4Type > 11
9502	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9503	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9504	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9505	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9506	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9507	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9508	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9509	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9510	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9511	}
9512	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9513	{
9514	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9515	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9516	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9517	\|\| pCtx->fs.Attr.n.u4Type > 11
9518	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9519	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9520	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9521	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9522	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9523	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9524	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9525	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9526	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9527	}
9528	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9529	{
9530	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9531	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9532	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9533	\|\| pCtx->gs.Attr.n.u4Type > 11
9534	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9535	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9536	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9537	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9538	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9539	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9540	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9541	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9542	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9543	}
9544	/* 64-bit capable CPUs. */
9545	#if HC_ARCH_BITS == 64
9546	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9547	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9548	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9549	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9550	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9551	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9552	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9553	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9554	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9555	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9556	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9557	#endif
9558	}
9559	else
9560	{
9561	/* V86 mode checks. */
9562	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9563	if (pVmcsInfo->RealMode.fRealOnV86Active)
9564	{
9565	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9566	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9567	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9568	}
9569	else
9570	{
9571	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9572	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9573	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9574	}
9575
9576	/* CS */
9577	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9578	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9579	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9580	/* SS */
9581	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9582	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9583	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9584	/* DS */
9585	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9586	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9587	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9588	/* ES */
9589	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9590	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9591	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9592	/* FS */
9593	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9594	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9595	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9596	/* GS */
9597	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9598	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9599	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9600	/* 64-bit capable CPUs. */
9601	#if HC_ARCH_BITS == 64
9602	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9603	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9604	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9605	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9606	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9607	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9608	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9609	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9610	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9611	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9612	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9613	#endif
9614	}
9615
9616	/*
9617	* TR.
9618	*/
9619	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9620	/* 64-bit capable CPUs. */
9621	#if HC_ARCH_BITS == 64
9622	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9623	#endif
9624	if (fLongModeGuest)
9625	{
9626	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9627	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9628	}
9629	else
9630	{
9631	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9632	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9633	VMX_IGS_TR_ATTR_TYPE_INVALID);
9634	}
9635	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9636	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9637	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9638	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9639	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9640	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9641	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9642	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9643
9644	/*
9645	* GDTR and IDTR.
9646	*/
9647	#if HC_ARCH_BITS == 64
9648	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9649	AssertRCBreak(rc);
9650	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9651
9652	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9653	AssertRCBreak(rc);
9654	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9655	#endif
9656
9657	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9658	AssertRCBreak(rc);
9659	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9660
9661	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9662	AssertRCBreak(rc);
9663	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9664
9665	/*
9666	* Guest Non-Register State.
9667	*/
9668	/* Activity State. */
9669	uint32_t u32ActivityState;
9670	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9671	AssertRCBreak(rc);
9672	HMVMX_CHECK_BREAK( !u32ActivityState
9673	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9674	VMX_IGS_ACTIVITY_STATE_INVALID);
9675	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9676	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9677	uint32_t u32IntrState;
9678	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9679	AssertRCBreak(rc);
9680	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9681	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9682	{
9683	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9684	}
9685
9686	/** @todo Activity state and injecting interrupts. Left as a todo since we
9687	* currently don't use activity states but ACTIVE. */
9688
9689	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9690	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9691
9692	/* Guest interruptibility-state. */
9693	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9694	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9695	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9696	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9697	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9698	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9699	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9700	if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
9701	{
9702	if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9703	{
9704	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9705	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9706	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9707	}
9708	else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9709	{
9710	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9711	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9712	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9713	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9714	}
9715	}
9716	/** @todo Assumes the processor is not in SMM. */
9717	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9718	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9719	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9720	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9721	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9722	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9723	&& VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9724	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9725	{
9726	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
9727	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9728	}
9729
9730	/* Pending debug exceptions. */
9731	#if HC_ARCH_BITS == 64
9732	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
9733	AssertRCBreak(rc);
9734	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9735	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9736	u32Val = u64Val; /* For pending debug exceptions checks below. */
9737	#else
9738	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
9739	AssertRCBreak(rc);
9740	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9741	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9742	#endif
9743
9744	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9745	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
9746	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9747	{
9748	if ( (u32Eflags & X86_EFL_TF)
9749	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9750	{
9751	/* Bit 14 is PendingDebug.BS. */
9752	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9753	}
9754	if ( !(u32Eflags & X86_EFL_TF)
9755	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9756	{
9757	/* Bit 14 is PendingDebug.BS. */
9758	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9759	}
9760	}
9761
9762	/* VMCS link pointer. */
9763	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9764	AssertRCBreak(rc);
9765	if (u64Val != UINT64_C(0xffffffffffffffff))
9766	{
9767	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9768	/** @todo Bits beyond the processor's physical-address width MBZ. */
9769	/** @todo 32-bit located in memory referenced by value of this field (as a
9770	* physical address) must contain the processor's VMCS revision ID. */
9771	/** @todo SMM checks. */
9772	}
9773
9774	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9775	* not using nested paging? */
9776	if ( pVM->hm.s.fNestedPaging
9777	&& !fLongModeGuest
9778	&& CPUMIsGuestInPAEModeEx(pCtx))
9779	{
9780	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9781	AssertRCBreak(rc);
9782	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9783
9784	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9785	AssertRCBreak(rc);
9786	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9787
9788	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9789	AssertRCBreak(rc);
9790	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9791
9792	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9793	AssertRCBreak(rc);
9794	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9795	}
9796
9797	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9798	if (uError == VMX_IGS_ERROR)
9799	uError = VMX_IGS_REASON_NOT_FOUND;
9800	} while (0);
9801
9802	pVCpu->hm.s.u32HMError = uError;
9803	return uError;
9804
9805	#undef HMVMX_ERROR_BREAK
9806	#undef HMVMX_CHECK_BREAK
9807	}
9808
9809
9810	/**
9811	* Setup the APIC-access page for virtualizing APIC access.
9812	*
9813	* This can cause a longjumps to R3 due to the acquisition of the PGM lock, hence
9814	* this not done as part of exporting guest state, see @bugref{8721}.
9815	*
9816	* @returns VBox status code.
9817	* @param pVCpu The cross context virtual CPU structure.
9818	*/
9819	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
9820	{
9821	PVM pVM = pVCpu->CTX_SUFF(pVM);
9822	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
9823
9824	Assert(PDMHasApic(pVM));
9825	Assert(u64MsrApicBase);
9826
9827	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
9828	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
9829
9830	/* Unalias any existing mapping. */
9831	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
9832	AssertRCReturn(rc, rc);
9833
9834	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
9835	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
9836	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
9837	AssertRCReturn(rc, rc);
9838
9839	/* Update the per-VCPU cache of the APIC base MSR. */
9840	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
9841	return VINF_SUCCESS;
9842	}
9843
9844
9845	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9846	/**
9847	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
9848	* nested-guest using hardware-assisted VMX.
9849	*
9850	* @param pVCpu The cross context virtual CPU structure.
9851	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
9852	* @param pVmcsInfoGst The guest VMCS info. object.
9853	*/
9854	static void hmR0VmxMergeMsrBitmapNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
9855	{
9856	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
9857	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
9858	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
9859	Assert(pu64MsrBitmapNstGst);
9860	Assert(pu64MsrBitmapGst);
9861	Assert(pu64MsrBitmap);
9862
9863	/*
9864	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
9865	* MSR that is intercepted by the guest is also intercepted while executing the
9866	* nested-guest using hardware-assisted VMX.
9867	*/
9868	uint32_t const cbFrag = sizeof(uint64_t);
9869	uint32_t const cFrags = X86_PAGE_4K_SIZE / cbFrag;
9870	for (uint32_t i = 0; i <= cFrags; i++)
9871	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
9872	}
9873
9874
9875	/**
9876	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
9877	* hardware-assisted VMX execution of the nested-guest.
9878	*
9879	* For a guest, we don't modify these controls once we set up the VMCS.
9880	*
9881	* For nested-guests since the guest hypervisor provides these controls on every
9882	* nested-guest VM-entry and could potentially change them everytime we need to
9883	* merge them before every nested-guest VM-entry.
9884	*
9885	* @returns VBox status code.
9886	* @param pVCpu The cross context virtual CPU structure.
9887	*/
9888	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
9889	{
9890	PVM pVM = pVCpu->CTX_SUFF(pVM);
9891	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
9892	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
9893	Assert(pVmcsNstGst);
9894
9895	/*
9896	* Merge the controls with the requirements of the guest VMCS.
9897	*
9898	* We do not need to validate the nested-guest VMX features specified in the
9899	* nested-guest VMCS with the features supported by the physical CPU as it's
9900	* already done by the VMLAUNCH/VMRESUME instruction emulation.
9901	*
9902	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the
9903	* guest are derived from the VMX features supported by the physical CPU.
9904	*/
9905
9906	/* Pin-based VM-execution controls. */
9907	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
9908
9909	/* Processor-based VM-execution controls. */
9910	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
9911	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
9912	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
9913	\| VMX_PROC_CTLS_USE_TPR_SHADOW
9914	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
9915
9916	/* Secondary processor-based VM-execution controls. */
9917	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
9918	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
9919	\| VMX_PROC_CTLS2_INVPCID
9920	\| VMX_PROC_CTLS2_RDTSCP
9921	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
9922	\| VMX_PROC_CTLS2_APIC_REG_VIRT
9923	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
9924	\| VMX_PROC_CTLS2_VMFUNC));
9925
9926	/*
9927	* VM-entry controls:
9928	* These controls contains state that depends on the nested-guest state (primarily
9929	* EFER MSR) and is thus not constant through VMLAUNCH/VMRESUME and the nested-guest
9930	* VM-exit. Although the nested-hypervisor cannot change it, we need to in order to
9931	* properly continue executing the nested-guest if the EFER MSR changes but does not
9932	* cause a nested-guest VM-exits.
9933	*
9934	* VM-exit controls:
9935	* These controls specify the host state on return. We cannot use the controls from
9936	* the nested-hypervisor state as is as it would contain the guest state rather than
9937	* the host state. Since the host state is subject to change (e.g. preemption, trips
9938	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
9939	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
9940	*
9941	* VM-entry MSR-load:
9942	* The guest MSRs from the VM-entry MSR-load area are already loaded into the
9943	* guest-CPU context by the VMLAUNCH/VMRESUME instruction emulation.
9944	*
9945	* VM-exit MSR-store:
9946	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU
9947	* context back into the VM-exit MSR-store area.
9948	*
9949	* VM-exit MSR-load areas:
9950	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence,
9951	* we can entirely ignore what the nested-hypervisor wants to load here.
9952	*/
9953
9954	/*
9955	* Exception bitmap.
9956	*
9957	* We could remove #UD from the guest bitmap and merge it with the nested-guest
9958	* bitmap here (and avoid doing anything while exporting nested-guest state), but to
9959	* keep the code more flexible if intercepting exceptions become more dynamic in
9960	* the future we do it as part of exporting the nested-guest state.
9961	*/
9962	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
9963
9964	/*
9965	* CR0/CR4 guest/host mask.
9966	*
9967	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest
9968	* must cause VM-exits, so we need to merge them here.
9969	*/
9970	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
9971	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
9972
9973	/*
9974	* Page-fault error-code mask and match.
9975	*
9976	* Although we require unrestricted guest execution (and thereby nested-paging) for
9977	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
9978	* normally intercept #PFs, it might intercept them for debugging purposes.
9979	*
9980	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF
9981	* filters. If the outer guest is intercepting #PFs we must intercept all #PFs.
9982	*/
9983	uint32_t u32XcptPFMask;
9984	uint32_t u32XcptPFMatch;
9985	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
9986	{
9987	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
9988	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
9989	}
9990	else
9991	{
9992	u32XcptPFMask = 0;
9993	u32XcptPFMatch = 0;
9994	}
9995
9996	/*
9997	* Pause-Loop exiting.
9998	*/
9999	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10000	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10001
10002	/*
10003	* I/O Bitmap.
10004	*
10005	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we
10006	* always intercept all I/O port accesses.
10007	*/
10008	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10009
10010	/*
10011	* APIC-access page.
10012	*
10013	* The APIC-access page address has already been initialized while setting up the
10014	* nested-guest VMCS. In theory, even if the guest-physical address is invalid, it
10015	* should not be on any consequence to the host or to the guest for that matter, but
10016	* we only accept valid addresses verified by the VMLAUNCH/VMRESUME instruction
10017	* emulation to keep it simple.
10018	*/
10019
10020	/*
10021	* Virtual-APIC page and TPR threshold.
10022	*
10023	* We shall use the host-physical address of the virtual-APIC page in guest memory directly.
10024	* For this reason, we can access the virtual-APIC page of the nested-guest only using
10025	* PGM physical handlers as we must not assume a kernel virtual-address mapping exists and
10026	* requesting PGM for a mapping could be expensive/resource intensive (PGM mapping cache).
10027	*/
10028	RTHCPHYS HCPhysVirtApic = NIL_RTHCPHYS;
10029	uint32_t const u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10030	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10031	{
10032	int rc = PGMPhysGCPhys2HCPhys(pVM, pVmcsNstGst->u64AddrVirtApic.u, &HCPhysVirtApic);
10033
10034	/*
10035	* If the guest hypervisor has loaded crap into the virtual-APIC page field
10036	* we would fail to obtain a valid host-physical address for its guest-physical
10037	* address.
10038	*
10039	* We currently do not support this scenario. Maybe in the future if there is a
10040	* pressing need we can explore making this particular set of conditions work.
10041	* Right now we just cause a VM-entry failure.
10042	*
10043	* This has already been checked by VMLAUNCH/VMRESUME instruction emulation,
10044	* so should not really failure at the moment.
10045	*/
10046	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10047	}
10048	else
10049	{
10050	/*
10051	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10052	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10053	* be taken care of by EPT/shadow paging.
10054	*/
10055	if (pVM->hm.s.fAllow64BitGuests)
10056	{
10057	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10058	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10059	}
10060	}
10061
10062	/*
10063	* Validate basic assumptions.
10064	*/
10065	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10066	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10067	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10068	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10069
10070	/*
10071	* Commit it to the nested-guest VMCS.
10072	*/
10073	int rc = VINF_SUCCESS;
10074	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10075	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10076	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10077	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10078	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10079	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10080	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10081	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10082	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10083	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10084	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10085	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10086	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10087	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10088	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10089	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10090	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10091	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10092	{
10093	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10094	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10095	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10096	}
10097	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10098	{
10099	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10100	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10101	}
10102	AssertRCReturn(rc, rc);
10103
10104	/*
10105	* Update the nested-guest VMCS cache.
10106	*/
10107	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10108	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10109	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10110	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10111	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10112	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10113	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10114	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10115	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10116
10117	/*
10118	* MSR bitmap.
10119	*
10120	* The MSR bitmap address has already been initialized while setting up the
10121	* nested-guest VMCS, here we need to merge the MSR bitmaps.
10122	*/
10123	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10124	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10125
10126	return VINF_SUCCESS;
10127	}
10128	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10129
10130
10131	/**
10132	* Does the preparations before executing guest code in VT-x.
10133	*
10134	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10135	* recompiler/IEM. We must be cautious what we do here regarding committing
10136	* guest-state information into the VMCS assuming we assuredly execute the
10137	* guest in VT-x mode.
10138	*
10139	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10140	* the common-state (TRPM/forceflags), we must undo those changes so that the
10141	* recompiler/IEM can (and should) use them when it resumes guest execution.
10142	* Otherwise such operations must be done when we can no longer exit to ring-3.
10143	*
10144	* @returns Strict VBox status code (i.e. informational status codes too).
10145	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10146	* have been disabled.
10147	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10148	* double-fault into the guest.
10149	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10150	* dispatched directly.
10151	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10152	*
10153	* @param pVCpu The cross context virtual CPU structure.
10154	* @param pVmxTransient The VMX-transient structure.
10155	* @param fStepping Whether we are single-stepping the guest in the
10156	* hypervisor debugger. Makes us ignore some of the reasons
10157	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10158	* if event dispatching took place.
10159	*/
10160	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10161	{
10162	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10163
10164	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10165	if (pVmxTransient->fIsNestedGuest)
10166	{
10167	RT_NOREF2(pVCpu, fStepping);
10168	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10169	return VINF_EM_RESCHEDULE_REM;
10170	}
10171	#endif
10172
10173	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10174	PGMRZDynMapFlushAutoSet(pVCpu);
10175	#endif
10176
10177	/*
10178	* Check and process force flag actions, some of which might require us to go back to ring-3.
10179	*/
10180	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10181	if (rcStrict == VINF_SUCCESS)
10182	{ /* FFs don't get set all the time. */ }
10183	else
10184	return rcStrict;
10185
10186	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10187	/*
10188	* Switch to the nested-guest VMCS as we may have transitioned into executing
10189	* the nested-guest without leaving ring-0. Otherwise, if we came from ring-3
10190	* we would load the nested-guest VMCS while entering the VMX ring-0 session.
10191	*
10192	* We do this as late as possible to minimize (though not completely remove)
10193	* clearing/loading VMCS again due to premature trips to ring-3 above.
10194	*/
10195	if (pVmxTransient->fIsNestedGuest)
10196	{
10197	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10198	{
10199	/*
10200	* Ensure we have synced everything from the guest VMCS and also flag that
10201	* that we need to export the full (nested) guest-CPU context to the
10202	* nested-guest VMCS.
10203	*/
10204	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
10205	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST);
10206
10207	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
10208	int rc = hmR0VmxSwitchVmcs(&pVCpu->hm.s.vmx.VmcsInfo, &pVCpu->hm.s.vmx.VmcsInfoNstGst);
10209	if (RT_LIKELY(rc == VINF_SUCCESS))
10210	{
10211	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = true;
10212	ASMSetFlags(fEFlags);
10213	pVmxTransient->pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10214
10215	/*
10216	* We use a different VM-exit MSR-store area for the nested-guest. Hence,
10217	* flag that we need to update the host MSR values there. Even if we decide
10218	* in the future to share the VM-exit MSR-store area page with the guest,
10219	* if its content differs, we would have to update the host MSRs anyway.
10220	*/
10221	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
10222	Assert(!pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer); /** @todo NSTVMX: Paranoia remove later. */
10223	}
10224	else
10225	{
10226	ASMSetFlags(fEFlags);
10227	return rc;
10228	}
10229	}
10230
10231	/*
10232	* Merge guest VMCS controls with the nested-guest VMCS controls.
10233	*
10234	* Even if we have not executed the guest prior to this (e.g. when resuming
10235	* from a saved state), we should be okay with merging controls as we
10236	* initialize the guest VMCS controls as part of VM setup phase.
10237	*/
10238	if (!pVCpu->hm.s.vmx.fMergedNstGstCtls)
10239	{
10240	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10241	AssertRCReturn(rc, rc);
10242	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10243	}
10244	}
10245	#endif
10246
10247	/*
10248	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10249	* We look at the guest VMCS control here as we always set it when supported by
10250	* the physical CPU. Looking at the nested-guest control here would not be
10251	* possible because they are not merged yet.
10252	*/
10253	PVM pVM = pVCpu->CTX_SUFF(pVM);
10254	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10255	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10256	&& (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10257	&& PDMHasApic(pVM))
10258	{
10259	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10260	AssertRCReturn(rc, rc);
10261	}
10262
10263	/*
10264	* Evaluate events to be injected into the guest.
10265	*
10266	* Events in TRPM can be injected without inspecting the guest state.
10267	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10268	* guest to cause a VM-exit the next time they are ready to receive the event.
10269	*/
10270	if (TRPMHasTrap(pVCpu))
10271	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10272
10273	uint32_t fIntrState;
10274	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10275
10276	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10277	/*
10278	* While evaluating pending events if something failed (unlikely) or if we were
10279	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10280	*/
10281	if ( rcStrict != VINF_SUCCESS
10282	\|\| ( pVmxTransient->fIsNestedGuest
10283	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)))
10284	return rcStrict;
10285	#endif
10286
10287	/*
10288	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10289	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10290	* also result in triple-faulting the VM.
10291	*
10292	* The above does not apply when executing a nested-guest (since unrestricted guest execution
10293	* is a requirement) regardless doing it avoid duplicating code elsewhere.
10294	*/
10295	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10296	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10297	{ /* likely */ }
10298	else
10299	{
10300	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10301	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10302	return rcStrict;
10303	}
10304
10305	/*
10306	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10307	* import CR3 themselves. We will need to update them here, as even as late as the above
10308	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10309	* the below force flags to be set.
10310	*/
10311	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10312	{
10313	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10314	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10315	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10316	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10317	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10318	}
10319	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10320	{
10321	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10322	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10323	}
10324
10325	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10326	/* Paranoia. */
10327	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10328	#endif
10329
10330	/*
10331	* No longjmps to ring-3 from this point on!!!
10332	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10333	* This also disables flushing of the R0-logger instance (if any).
10334	*/
10335	VMMRZCallRing3Disable(pVCpu);
10336
10337	/*
10338	* Export the guest state bits.
10339	*
10340	* We cannot perform longjmps while loading the guest state because we do not preserve the
10341	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10342	* CPU migration.
10343	*
10344	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10345	* registers. Hence, loading of the guest state needs to be done -after- injection of events.
10346	*/
10347	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10348	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10349	{ /* likely */ }
10350	else
10351	{
10352	VMMRZCallRing3Enable(pVCpu);
10353	return rcStrict;
10354	}
10355
10356	/*
10357	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10358	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10359	* preemption disabled for a while. Since this is purely to aid the
10360	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10361	* disable interrupt on NT.
10362	*
10363	* We need to check for force-flags that could've possible been altered since we last
10364	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10365	* see @bugref{6398}).
10366	*
10367	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10368	* to ring-3 before executing guest code.
10369	*/
10370	pVmxTransient->fEFlags = ASMIntDisableFlags();
10371
10372	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10373	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10374	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10375	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10376	{
10377	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10378	{
10379	pVCpu->hm.s.Event.fPending = false;
10380
10381	/*
10382	* We've injected any pending events. This is really the point of no return (to ring-3).
10383	*
10384	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10385	* returns from this function, so don't enable them here.
10386	*/
10387	return VINF_SUCCESS;
10388	}
10389
10390	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10391	rcStrict = VINF_EM_RAW_INTERRUPT;
10392	}
10393	else
10394	{
10395	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10396	rcStrict = VINF_EM_RAW_TO_R3;
10397	}
10398
10399	ASMSetFlags(pVmxTransient->fEFlags);
10400	VMMRZCallRing3Enable(pVCpu);
10401
10402	return rcStrict;
10403	}
10404
10405
10406	/**
10407	* Final preparations before executing guest code using hardware-assisted VMX.
10408	*
10409	* We can no longer get preempted to a different host CPU and there are no returns
10410	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10411	* failures), this function is not intended to fail sans unrecoverable hardware
10412	* errors.
10413	*
10414	* @param pVCpu The cross context virtual CPU structure.
10415	* @param pVmxTransient The VMX-transient structure.
10416	*
10417	* @remarks Called with preemption disabled.
10418	* @remarks No-long-jump zone!!!
10419	*/
10420	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10421	{
10422	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10423	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10424	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10425	Assert(!pVCpu->hm.s.Event.fPending);
10426
10427	/*
10428	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10429	*/
10430	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10431	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10432
10433	PVM pVM = pVCpu->CTX_SUFF(pVM);
10434	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10435
10436	if (!CPUMIsGuestFPUStateActive(pVCpu))
10437	{
10438	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10439	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10440	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10441	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10442	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10443	}
10444
10445	/*
10446	* Re-save the host state bits as we may've been preempted (only happens when
10447	* thread-context hooks are used or when the VM start function changes).
10448	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10449	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10450	* see @bugref{8432}.
10451	*
10452	* This may also happen when switching to/from a nested-guest VMCS without leaving
10453	* ring-0.
10454	*/
10455	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10456	{
10457	int rc = hmR0VmxExportHostState(pVCpu);
10458	AssertRC(rc);
10459	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
10460	}
10461	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10462
10463	/*
10464	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10465	*/
10466	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10467	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10468	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10469
10470	/*
10471	* Store status of the shared guest/host debug state at the time of VM-entry.
10472	*/
10473	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
10474	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
10475	{
10476	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
10477	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
10478	}
10479	else
10480	#endif
10481	{
10482	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10483	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10484	}
10485
10486	/*
10487	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10488	* more than one conditional check. The post-run side of our code shall determine
10489	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10490	*/
10491	if (pVmcsInfo->pbVirtApic)
10492	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10493
10494	/*
10495	* Update the host MSRs values in the VM-exit MSR-load area.
10496	*/
10497	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10498	{
10499	if (pVmcsInfo->cExitMsrLoad > 0)
10500	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10501	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10502	}
10503
10504	/*
10505	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10506	* VMX-preemption timer based on the next virtual sync clock deadline.
10507	*/
10508	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10509	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10510	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10511	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10512	{
10513	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10514	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10515	}
10516
10517	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10518	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10519	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10520	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
10521
10522	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10523
10524	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10525	as we're about to start executing the guest . */
10526
10527	/*
10528	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10529	*
10530	* This is done this late as updating the TSC offsetting/preemption timer above
10531	* figures out if we can skip intercepting RDTSCP by calculating the number of
10532	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10533	*/
10534	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10535	{
10536	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10537	{
10538	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10539	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10540	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10541	AssertRC(rc);
10542	}
10543	else
10544	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
10545	}
10546
10547	#ifdef VBOX_STRICT
10548	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo);
10549	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10550	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo));
10551	#endif
10552
10553	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10554	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10555	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs. */
10556	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10557	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10558	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10559	#endif
10560	}
10561
10562
10563	/**
10564	* First C routine invoked after running guest code using hardware-assisted VMX.
10565	*
10566	* @param pVCpu The cross context virtual CPU structure.
10567	* @param pVmxTransient The VMX-transient structure.
10568	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10569	*
10570	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10571	*
10572	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10573	* unconditionally when it is safe to do so.
10574	*/
10575	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10576	{
10577	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10578
10579	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10580	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10581	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10582	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10583	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10584	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10585
10586	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10587	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10588	{
10589	uint64_t uGstTsc;
10590	if (!pVmxTransient->fIsNestedGuest)
10591	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10592	else
10593	{
10594	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10595	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10596	}
10597	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10598	}
10599
10600	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10601	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
10602	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10603
10604	#if HC_ARCH_BITS == 64
10605	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10606	#endif
10607	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
10608	/* The 64-on-32 switcher maintains VMCS-launch state on its own
10609	and we need to leave it alone here. */
10610	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
10611	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10612	#else
10613	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10614	#endif
10615	#ifdef VBOX_STRICT
10616	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10617	#endif
10618	Assert(!ASMIntAreEnabled());
10619	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10620	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10621
10622	/*
10623	* Save the basic VM-exit reason and check if the VM-entry failed.
10624	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10625	*/
10626	uint32_t uExitReason;
10627	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
10628	AssertRC(rc);
10629	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
10630	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
10631
10632	/*
10633	* Check if VMLAUNCH/VMRESUME succeeded.
10634	* If this failed, we cause a guru meditation and cease further execution.
10635	*/
10636	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
10637	{
10638	/*
10639	* Update the VM-exit history array here even if the VM-entry failed due to:
10640	* - Invalid guest state.
10641	* - MSR loading.
10642	* - Machine-check event.
10643	*
10644	* In any of the above cases we will still have a "valid" VM-exit reason
10645	* despite @a fVMEntryFailed being false.
10646	*
10647	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
10648	*
10649	* Note! We don't have CS or RIP at this point. Will probably address that later
10650	* by amending the history entry added here.
10651	*/
10652	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
10653	UINT64_MAX, uHostTsc);
10654
10655	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
10656	{
10657	VMMRZCallRing3Enable(pVCpu);
10658
10659	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10660	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10661
10662	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
10663	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10664	AssertRC(rc);
10665	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
10666	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
10667	AssertRC(rc);
10668	#else
10669	/*
10670	* Import the guest-interruptibility state always as we need it while evaluating
10671	* injecting events on re-entry.
10672	*
10673	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
10674	* checking for real-mode while exporting the state because all bits that cause
10675	* mode changes wrt CR0 are intercepted.
10676	*/
10677	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
10678	AssertRC(rc);
10679	#endif
10680
10681	/*
10682	* Sync the TPR shadow with our APIC state.
10683	*/
10684	if ( !pVmxTransient->fIsNestedGuest
10685	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
10686	{
10687	Assert(pVmcsInfo->pbVirtApic);
10688	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
10689	{
10690	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
10691	AssertRC(rc);
10692	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
10693	}
10694	}
10695
10696	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10697	return;
10698	}
10699	}
10700	else
10701	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
10702
10703	VMMRZCallRing3Enable(pVCpu);
10704	}
10705
10706
10707	/**
10708	* Runs the guest code using hardware-assisted VMX the normal way.
10709	*
10710	* @returns VBox status code.
10711	* @param pVCpu The cross context virtual CPU structure.
10712	* @param pcLoops Pointer to the number of executed loops.
10713	*/
10714	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
10715	{
10716	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10717	Assert(pcLoops);
10718	Assert(*pcLoops <= cMaxResumeLoops);
10719
10720	VMXTRANSIENT VmxTransient;
10721	RT_ZERO(VmxTransient);
10722	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10723
10724	/* Paranoia. */
10725	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
10726	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10727
10728	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10729	for (;;)
10730	{
10731	Assert(!HMR0SuspendPending());
10732	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10733	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10734
10735	/*
10736	* Preparatory work for running nested-guest code, this may force us to
10737	* return to ring-3.
10738	*
10739	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10740	*/
10741	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10742	if (rcStrict != VINF_SUCCESS)
10743	break;
10744
10745	/* Interrupts are disabled at this point! */
10746	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10747	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10748	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10749	/* Interrupts are re-enabled at this point! */
10750
10751	/*
10752	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10753	*/
10754	if (RT_SUCCESS(rcRun))
10755	{ /* very likely */ }
10756	else
10757	{
10758	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10759	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10760	return rcRun;
10761	}
10762
10763	/*
10764	* Profile the VM-exit.
10765	*/
10766	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10767	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10768	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10769	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10770	HMVMX_START_EXIT_DISPATCH_PROF();
10771
10772	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10773
10774	/*
10775	* Handle the VM-exit.
10776	*/
10777	#ifdef HMVMX_USE_FUNCTION_TABLE
10778	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
10779	#else
10780	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
10781	#endif
10782	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10783	if (rcStrict == VINF_SUCCESS)
10784	{
10785	if (++(*pcLoops) <= cMaxResumeLoops)
10786	continue;
10787	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10788	rcStrict = VINF_EM_RAW_INTERRUPT;
10789	}
10790	break;
10791	}
10792
10793	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10794	return rcStrict;
10795	}
10796
10797	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10798	/**
10799	* Runs the nested-guest code using hardware-assisted VMX.
10800	*
10801	* @returns VBox status code.
10802	* @param pVCpu The cross context virtual CPU structure.
10803	* @param pcLoops Pointer to the number of executed loops.
10804	*
10805	* @sa hmR0VmxRunGuestCodeNormal().
10806	*/
10807	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
10808	{
10809	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10810	Assert(pcLoops);
10811	Assert(*pcLoops <= cMaxResumeLoops);
10812
10813	VMXTRANSIENT VmxTransient;
10814	RT_ZERO(VmxTransient);
10815	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10816	VmxTransient.fIsNestedGuest = true;
10817
10818	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10819	for (;;)
10820	{
10821	Assert(!HMR0SuspendPending());
10822	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10823	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10824
10825	/*
10826	* Preparatory work for running guest code, this may force us to
10827	* return to ring-3.
10828	*
10829	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10830	*/
10831	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10832	if (rcStrict != VINF_SUCCESS)
10833	break;
10834
10835	/* Interrupts are disabled at this point! */
10836	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10837	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10838	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10839	/* Interrupts are re-enabled at this point! */
10840
10841	/*
10842	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10843	*/
10844	if (RT_SUCCESS(rcRun))
10845	{ /* very likely */ }
10846	else
10847	{
10848	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10849	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10850	return rcRun;
10851	}
10852
10853	/*
10854	* Profile the VM-exit.
10855	*/
10856	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10857	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10858	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10859	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10860	HMVMX_START_EXIT_DISPATCH_PROF();
10861
10862	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10863
10864	/*
10865	* Handle the VM-exit.
10866	*/
10867	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
10868	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10869	if ( rcStrict == VINF_SUCCESS
10870	&& CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10871	{
10872	if (++(*pcLoops) <= cMaxResumeLoops)
10873	continue;
10874	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10875	rcStrict = VINF_EM_RAW_INTERRUPT;
10876	}
10877	break;
10878	}
10879
10880	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10881	return rcStrict;
10882	}
10883	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10884
10885
10886	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
10887	* probes.
10888	*
10889	* The following few functions and associated structure contains the bloat
10890	* necessary for providing detailed debug events and dtrace probes as well as
10891	* reliable host side single stepping. This works on the principle of
10892	* "subclassing" the normal execution loop and workers. We replace the loop
10893	* method completely and override selected helpers to add necessary adjustments
10894	* to their core operation.
10895	*
10896	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
10897	* any performance for debug and analysis features.
10898	*
10899	* @{
10900	*/
10901
10902	/**
10903	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
10904	* the debug run loop.
10905	*/
10906	typedef struct VMXRUNDBGSTATE
10907	{
10908	/** The RIP we started executing at. This is for detecting that we stepped. */
10909	uint64_t uRipStart;
10910	/** The CS we started executing with. */
10911	uint16_t uCsStart;
10912
10913	/** Whether we've actually modified the 1st execution control field. */
10914	bool fModifiedProcCtls : 1;
10915	/** Whether we've actually modified the 2nd execution control field. */
10916	bool fModifiedProcCtls2 : 1;
10917	/** Whether we've actually modified the exception bitmap. */
10918	bool fModifiedXcptBitmap : 1;
10919
10920	/** We desire the modified the CR0 mask to be cleared. */
10921	bool fClearCr0Mask : 1;
10922	/** We desire the modified the CR4 mask to be cleared. */
10923	bool fClearCr4Mask : 1;
10924	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
10925	uint32_t fCpe1Extra;
10926	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
10927	uint32_t fCpe1Unwanted;
10928	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
10929	uint32_t fCpe2Extra;
10930	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
10931	uint32_t bmXcptExtra;
10932	/** The sequence number of the Dtrace provider settings the state was
10933	* configured against. */
10934	uint32_t uDtraceSettingsSeqNo;
10935	/** VM-exits to check (one bit per VM-exit). */
10936	uint32_t bmExitsToCheck[3];
10937
10938	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
10939	uint32_t fProcCtlsInitial;
10940	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
10941	uint32_t fProcCtls2Initial;
10942	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
10943	uint32_t bmXcptInitial;
10944	} VMXRUNDBGSTATE;
10945	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
10946	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
10947
10948
10949	/**
10950	* Initializes the VMXRUNDBGSTATE structure.
10951	*
10952	* @param pVCpu The cross context virtual CPU structure of the
10953	* calling EMT.
10954	* @param pVmxTransient The VMX-transient structure.
10955	* @param pDbgState The debug state to initialize.
10956	*/
10957	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
10958	{
10959	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
10960	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
10961
10962	pDbgState->fModifiedProcCtls = false;
10963	pDbgState->fModifiedProcCtls2 = false;
10964	pDbgState->fModifiedXcptBitmap = false;
10965	pDbgState->fClearCr0Mask = false;
10966	pDbgState->fClearCr4Mask = false;
10967	pDbgState->fCpe1Extra = 0;
10968	pDbgState->fCpe1Unwanted = 0;
10969	pDbgState->fCpe2Extra = 0;
10970	pDbgState->bmXcptExtra = 0;
10971	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
10972	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
10973	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
10974	}
10975
10976
10977	/**
10978	* Updates the VMSC fields with changes requested by @a pDbgState.
10979	*
10980	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
10981	* immediately before executing guest code, i.e. when interrupts are disabled.
10982	* We don't check status codes here as we cannot easily assert or return in the
10983	* latter case.
10984	*
10985	* @param pVCpu The cross context virtual CPU structure.
10986	* @param pVmxTransient The VMX-transient structure.
10987	* @param pDbgState The debug state.
10988	*/
10989	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
10990	{
10991	/*
10992	* Ensure desired flags in VMCS control fields are set.
10993	* (Ignoring write failure here, as we're committed and it's just debug extras.)
10994	*
10995	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
10996	* there should be no stale data in pCtx at this point.
10997	*/
10998	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10999	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11000	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11001	{
11002	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11003	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11004	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11005	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11006	pDbgState->fModifiedProcCtls = true;
11007	}
11008
11009	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11010	{
11011	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11012	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11013	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11014	pDbgState->fModifiedProcCtls2 = true;
11015	}
11016
11017	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11018	{
11019	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11020	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11021	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11022	pDbgState->fModifiedXcptBitmap = true;
11023	}
11024
11025	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11026	{
11027	pVmcsInfo->u64Cr0Mask = 0;
11028	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
11029	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11030	}
11031
11032	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11033	{
11034	pVmcsInfo->u64Cr4Mask = 0;
11035	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
11036	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11037	}
11038
11039	NOREF(pVCpu);
11040	}
11041
11042
11043	/**
11044	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11045	* re-entry next time around.
11046	*
11047	* @returns Strict VBox status code (i.e. informational status codes too).
11048	* @param pVCpu The cross context virtual CPU structure.
11049	* @param pVmxTransient The VMX-transient structure.
11050	* @param pDbgState The debug state.
11051	* @param rcStrict The return code from executing the guest using single
11052	* stepping.
11053	*/
11054	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11055	VBOXSTRICTRC rcStrict)
11056	{
11057	/*
11058	* Restore VM-exit control settings as we may not reenter this function the
11059	* next time around.
11060	*/
11061	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11062
11063	/* We reload the initial value, trigger what we can of recalculations the
11064	next time around. From the looks of things, that's all that's required atm. */
11065	if (pDbgState->fModifiedProcCtls)
11066	{
11067	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11068	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11069	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11070	AssertRCReturn(rc2, rc2);
11071	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11072	}
11073
11074	/* We're currently the only ones messing with this one, so just restore the
11075	cached value and reload the field. */
11076	if ( pDbgState->fModifiedProcCtls2
11077	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11078	{
11079	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11080	AssertRCReturn(rc2, rc2);
11081	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11082	}
11083
11084	/* If we've modified the exception bitmap, we restore it and trigger
11085	reloading and partial recalculation the next time around. */
11086	if (pDbgState->fModifiedXcptBitmap)
11087	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11088
11089	return rcStrict;
11090	}
11091
11092
11093	/**
11094	* Configures VM-exit controls for current DBGF and DTrace settings.
11095	*
11096	* This updates @a pDbgState and the VMCS execution control fields to reflect
11097	* the necessary VM-exits demanded by DBGF and DTrace.
11098	*
11099	* @param pVCpu The cross context virtual CPU structure.
11100	* @param pVmxTransient The VMX-transient structure. May update
11101	* fUpdatedTscOffsettingAndPreemptTimer.
11102	* @param pDbgState The debug state.
11103	*/
11104	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11105	{
11106	/*
11107	* Take down the dtrace serial number so we can spot changes.
11108	*/
11109	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11110	ASMCompilerBarrier();
11111
11112	/*
11113	* We'll rebuild most of the middle block of data members (holding the
11114	* current settings) as we go along here, so start by clearing it all.
11115	*/
11116	pDbgState->bmXcptExtra = 0;
11117	pDbgState->fCpe1Extra = 0;
11118	pDbgState->fCpe1Unwanted = 0;
11119	pDbgState->fCpe2Extra = 0;
11120	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11121	pDbgState->bmExitsToCheck[i] = 0;
11122
11123	/*
11124	* Software interrupts (INT XXh) - no idea how to trigger these...
11125	*/
11126	PVM pVM = pVCpu->CTX_SUFF(pVM);
11127	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11128	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11129	{
11130	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11131	}
11132
11133	/*
11134	* INT3 breakpoints - triggered by #BP exceptions.
11135	*/
11136	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11137	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11138
11139	/*
11140	* Exception bitmap and XCPT events+probes.
11141	*/
11142	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11143	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11144	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11145
11146	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11147	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11148	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11149	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11150	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11151	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11152	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11153	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11154	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11155	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11156	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11157	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11158	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11159	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11160	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11161	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11162	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11163	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11164
11165	if (pDbgState->bmXcptExtra)
11166	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11167
11168	/*
11169	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11170	*
11171	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11172	* So, when adding/changing/removing please don't forget to update it.
11173	*
11174	* Some of the macros are picking up local variables to save horizontal space,
11175	* (being able to see it in a table is the lesser evil here).
11176	*/
11177	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11178	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11179	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11180	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11181	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11182	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11183	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11184	} else do { } while (0)
11185	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11186	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11187	{ \
11188	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11189	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11190	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11191	} else do { } while (0)
11192	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11193	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11194	{ \
11195	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11196	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11197	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11198	} else do { } while (0)
11199	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11200	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11201	{ \
11202	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11203	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11204	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11205	} else do { } while (0)
11206
11207	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11208	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11209	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11210	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11211	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11212
11213	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11214	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11215	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11216	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11217	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11218	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11219	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11220	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11221	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11222	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11223	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11224	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11225	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11226	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11227	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11228	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11229	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11230	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11231	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11232	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11233	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11234	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11235	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11236	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11237	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11238	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11239	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11240	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11241	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11242	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11243	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11244	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11245	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11246	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11247	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11248	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11249
11250	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11251	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11252	{
11253	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11254	\| CPUMCTX_EXTRN_APIC_TPR);
11255	AssertRC(rc);
11256
11257	#if 0 /** @todo fix me */
11258	pDbgState->fClearCr0Mask = true;
11259	pDbgState->fClearCr4Mask = true;
11260	#endif
11261	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11262	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11263	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11264	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11265	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11266	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11267	require clearing here and in the loop if we start using it. */
11268	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11269	}
11270	else
11271	{
11272	if (pDbgState->fClearCr0Mask)
11273	{
11274	pDbgState->fClearCr0Mask = false;
11275	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11276	}
11277	if (pDbgState->fClearCr4Mask)
11278	{
11279	pDbgState->fClearCr4Mask = false;
11280	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11281	}
11282	}
11283	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11284	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11285
11286	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11287	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11288	{
11289	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11290	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11291	}
11292	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11293	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11294
11295	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11296	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11297	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11298	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11299	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11300	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11301	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11302	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11303	#if 0 /** @todo too slow, fix handler. */
11304	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11305	#endif
11306	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11307
11308	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11309	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11310	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11311	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11312	{
11313	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11314	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11315	}
11316	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11317	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11318	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11319	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11320
11321	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11322	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11323	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11324	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11325	{
11326	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11327	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11328	}
11329	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11330	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11331	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11332	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11333
11334	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11335	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11336	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11337	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11338	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11339	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11340	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11341	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11342	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11343	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11344	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11345	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11346	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11347	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11348	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11349	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11350	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11351	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11352	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11353	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11354	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11355	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11356
11357	#undef IS_EITHER_ENABLED
11358	#undef SET_ONLY_XBM_IF_EITHER_EN
11359	#undef SET_CPE1_XBM_IF_EITHER_EN
11360	#undef SET_CPEU_XBM_IF_EITHER_EN
11361	#undef SET_CPE2_XBM_IF_EITHER_EN
11362
11363	/*
11364	* Sanitize the control stuff.
11365	*/
11366	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11367	if (pDbgState->fCpe2Extra)
11368	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11369	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11370	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11371	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11372	{
11373	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11374	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11375	}
11376
11377	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11378	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11379	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11380	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11381	}
11382
11383
11384	/**
11385	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11386	* appropriate.
11387	*
11388	* The caller has checked the VM-exit against the
11389	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11390	* already, so we don't have to do that either.
11391	*
11392	* @returns Strict VBox status code (i.e. informational status codes too).
11393	* @param pVCpu The cross context virtual CPU structure.
11394	* @param pVmxTransient The VMX-transient structure.
11395	* @param uExitReason The VM-exit reason.
11396	*
11397	* @remarks The name of this function is displayed by dtrace, so keep it short
11398	* and to the point. No longer than 33 chars long, please.
11399	*/
11400	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11401	{
11402	/*
11403	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11404	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11405	*
11406	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11407	* does. Must add/change/remove both places. Same ordering, please.
11408	*
11409	* Added/removed events must also be reflected in the next section
11410	* where we dispatch dtrace events.
11411	*/
11412	bool fDtrace1 = false;
11413	bool fDtrace2 = false;
11414	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11415	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11416	uint32_t uEventArg = 0;
11417	#define SET_EXIT(a_EventSubName) \
11418	do { \
11419	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11420	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11421	} while (0)
11422	#define SET_BOTH(a_EventSubName) \
11423	do { \
11424	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11425	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11426	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11427	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11428	} while (0)
11429	switch (uExitReason)
11430	{
11431	case VMX_EXIT_MTF:
11432	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11433
11434	case VMX_EXIT_XCPT_OR_NMI:
11435	{
11436	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11437	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11438	{
11439	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11440	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11441	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11442	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11443	{
11444	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11445	{
11446	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11447	uEventArg = pVmxTransient->uExitIntErrorCode;
11448	}
11449	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11450	switch (enmEvent1)
11451	{
11452	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11453	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11454	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11455	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11456	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11457	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11458	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11459	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11460	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11461	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11462	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11463	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11464	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11465	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11466	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11467	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11468	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11469	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11470	default: break;
11471	}
11472	}
11473	else
11474	AssertFailed();
11475	break;
11476
11477	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11478	uEventArg = idxVector;
11479	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11480	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11481	break;
11482	}
11483	break;
11484	}
11485
11486	case VMX_EXIT_TRIPLE_FAULT:
11487	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11488	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11489	break;
11490	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11491	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11492	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11493	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11494	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11495
11496	/* Instruction specific VM-exits: */
11497	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11498	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11499	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11500	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11501	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11502	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11503	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11504	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11505	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11506	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11507	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11508	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11509	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11510	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11511	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11512	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11513	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11514	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11515	case VMX_EXIT_MOV_CRX:
11516	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11517	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11518	SET_BOTH(CRX_READ);
11519	else
11520	SET_BOTH(CRX_WRITE);
11521	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11522	break;
11523	case VMX_EXIT_MOV_DRX:
11524	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11525	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11526	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11527	SET_BOTH(DRX_READ);
11528	else
11529	SET_BOTH(DRX_WRITE);
11530	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11531	break;
11532	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11533	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11534	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11535	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11536	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11537	case VMX_EXIT_GDTR_IDTR_ACCESS:
11538	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11539	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11540	{
11541	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11542	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11543	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11544	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11545	}
11546	break;
11547
11548	case VMX_EXIT_LDTR_TR_ACCESS:
11549	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11550	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11551	{
11552	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11553	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11554	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11555	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11556	}
11557	break;
11558
11559	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11560	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11561	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11562	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11563	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11564	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11565	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11566	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11567	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11568	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11569	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11570
11571	/* Events that aren't relevant at this point. */
11572	case VMX_EXIT_EXT_INT:
11573	case VMX_EXIT_INT_WINDOW:
11574	case VMX_EXIT_NMI_WINDOW:
11575	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11576	case VMX_EXIT_PREEMPT_TIMER:
11577	case VMX_EXIT_IO_INSTR:
11578	break;
11579
11580	/* Errors and unexpected events. */
11581	case VMX_EXIT_INIT_SIGNAL:
11582	case VMX_EXIT_SIPI:
11583	case VMX_EXIT_IO_SMI:
11584	case VMX_EXIT_SMI:
11585	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11586	case VMX_EXIT_ERR_MSR_LOAD:
11587	case VMX_EXIT_ERR_MACHINE_CHECK:
11588	break;
11589
11590	default:
11591	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11592	break;
11593	}
11594	#undef SET_BOTH
11595	#undef SET_EXIT
11596
11597	/*
11598	* Dtrace tracepoints go first. We do them here at once so we don't
11599	* have to copy the guest state saving and stuff a few dozen times.
11600	* Down side is that we've got to repeat the switch, though this time
11601	* we use enmEvent since the probes are a subset of what DBGF does.
11602	*/
11603	if (fDtrace1 \|\| fDtrace2)
11604	{
11605	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11606	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11607	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11608	switch (enmEvent1)
11609	{
11610	/** @todo consider which extra parameters would be helpful for each probe. */
11611	case DBGFEVENT_END: break;
11612	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11613	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11614	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11615	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11616	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11617	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11618	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11619	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11620	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
11621	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
11622	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
11623	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
11624	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
11625	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
11626	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
11627	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
11628	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
11629	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
11630	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11631	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11632	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
11633	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
11634	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
11635	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
11636	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
11637	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
11638	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
11639	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11640	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11641	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11642	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11643	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11644	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
11645	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11646	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
11647	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
11648	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
11649	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
11650	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
11651	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
11652	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
11653	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
11654	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
11655	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
11656	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
11657	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
11658	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
11659	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
11660	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
11661	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
11662	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
11663	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
11664	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
11665	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
11666	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
11667	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
11668	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
11669	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
11670	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
11671	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
11672	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
11673	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
11674	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
11675	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
11676	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
11677	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
11678	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
11679	}
11680	switch (enmEvent2)
11681	{
11682	/** @todo consider which extra parameters would be helpful for each probe. */
11683	case DBGFEVENT_END: break;
11684	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
11685	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11686	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
11687	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
11688	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
11689	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
11690	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
11691	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
11692	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
11693	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11694	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11695	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11696	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11697	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11698	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
11699	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11700	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
11701	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
11702	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
11703	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
11704	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
11705	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
11706	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
11707	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
11708	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
11709	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
11710	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
11711	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
11712	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
11713	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
11714	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
11715	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
11716	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
11717	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
11718	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
11719	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
11720	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
11721	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
11722	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
11723	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
11724	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
11725	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
11726	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
11727	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
11728	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
11729	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
11730	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
11731	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
11732	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
11733	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
11734	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
11735	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
11736	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
11737	}
11738	}
11739
11740	/*
11741	* Fire of the DBGF event, if enabled (our check here is just a quick one,
11742	* the DBGF call will do a full check).
11743	*
11744	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
11745	* Note! If we have to events, we prioritize the first, i.e. the instruction
11746	* one, in order to avoid event nesting.
11747	*/
11748	PVM pVM = pVCpu->CTX_SUFF(pVM);
11749	if ( enmEvent1 != DBGFEVENT_END
11750	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
11751	{
11752	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11753	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
11754	if (rcStrict != VINF_SUCCESS)
11755	return rcStrict;
11756	}
11757	else if ( enmEvent2 != DBGFEVENT_END
11758	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
11759	{
11760	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11761	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
11762	if (rcStrict != VINF_SUCCESS)
11763	return rcStrict;
11764	}
11765
11766	return VINF_SUCCESS;
11767	}
11768
11769
11770	/**
11771	* Single-stepping VM-exit filtering.
11772	*
11773	* This is preprocessing the VM-exits and deciding whether we've gotten far
11774	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
11775	* handling is performed.
11776	*
11777	* @returns Strict VBox status code (i.e. informational status codes too).
11778	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
11779	* @param pVmxTransient The VMX-transient structure.
11780	* @param pDbgState The debug state.
11781	*/
11782	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11783	{
11784	/*
11785	* Expensive (saves context) generic dtrace VM-exit probe.
11786	*/
11787	uint32_t const uExitReason = pVmxTransient->uExitReason;
11788	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
11789	{ /* more likely */ }
11790	else
11791	{
11792	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11793	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11794	AssertRC(rc);
11795	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
11796	}
11797
11798	/*
11799	* Check for host NMI, just to get that out of the way.
11800	*/
11801	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
11802	{ /* normally likely */ }
11803	else
11804	{
11805	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11806	AssertRCReturn(rc2, rc2);
11807	uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11808	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11809	return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
11810	}
11811
11812	/*
11813	* Check for single stepping event if we're stepping.
11814	*/
11815	if (pVCpu->hm.s.fSingleInstruction)
11816	{
11817	switch (uExitReason)
11818	{
11819	case VMX_EXIT_MTF:
11820	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11821
11822	/* Various events: */
11823	case VMX_EXIT_XCPT_OR_NMI:
11824	case VMX_EXIT_EXT_INT:
11825	case VMX_EXIT_TRIPLE_FAULT:
11826	case VMX_EXIT_INT_WINDOW:
11827	case VMX_EXIT_NMI_WINDOW:
11828	case VMX_EXIT_TASK_SWITCH:
11829	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11830	case VMX_EXIT_APIC_ACCESS:
11831	case VMX_EXIT_EPT_VIOLATION:
11832	case VMX_EXIT_EPT_MISCONFIG:
11833	case VMX_EXIT_PREEMPT_TIMER:
11834
11835	/* Instruction specific VM-exits: */
11836	case VMX_EXIT_CPUID:
11837	case VMX_EXIT_GETSEC:
11838	case VMX_EXIT_HLT:
11839	case VMX_EXIT_INVD:
11840	case VMX_EXIT_INVLPG:
11841	case VMX_EXIT_RDPMC:
11842	case VMX_EXIT_RDTSC:
11843	case VMX_EXIT_RSM:
11844	case VMX_EXIT_VMCALL:
11845	case VMX_EXIT_VMCLEAR:
11846	case VMX_EXIT_VMLAUNCH:
11847	case VMX_EXIT_VMPTRLD:
11848	case VMX_EXIT_VMPTRST:
11849	case VMX_EXIT_VMREAD:
11850	case VMX_EXIT_VMRESUME:
11851	case VMX_EXIT_VMWRITE:
11852	case VMX_EXIT_VMXOFF:
11853	case VMX_EXIT_VMXON:
11854	case VMX_EXIT_MOV_CRX:
11855	case VMX_EXIT_MOV_DRX:
11856	case VMX_EXIT_IO_INSTR:
11857	case VMX_EXIT_RDMSR:
11858	case VMX_EXIT_WRMSR:
11859	case VMX_EXIT_MWAIT:
11860	case VMX_EXIT_MONITOR:
11861	case VMX_EXIT_PAUSE:
11862	case VMX_EXIT_GDTR_IDTR_ACCESS:
11863	case VMX_EXIT_LDTR_TR_ACCESS:
11864	case VMX_EXIT_INVEPT:
11865	case VMX_EXIT_RDTSCP:
11866	case VMX_EXIT_INVVPID:
11867	case VMX_EXIT_WBINVD:
11868	case VMX_EXIT_XSETBV:
11869	case VMX_EXIT_RDRAND:
11870	case VMX_EXIT_INVPCID:
11871	case VMX_EXIT_VMFUNC:
11872	case VMX_EXIT_RDSEED:
11873	case VMX_EXIT_XSAVES:
11874	case VMX_EXIT_XRSTORS:
11875	{
11876	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11877	AssertRCReturn(rc, rc);
11878	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
11879	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
11880	return VINF_EM_DBG_STEPPED;
11881	break;
11882	}
11883
11884	/* Errors and unexpected events: */
11885	case VMX_EXIT_INIT_SIGNAL:
11886	case VMX_EXIT_SIPI:
11887	case VMX_EXIT_IO_SMI:
11888	case VMX_EXIT_SMI:
11889	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11890	case VMX_EXIT_ERR_MSR_LOAD:
11891	case VMX_EXIT_ERR_MACHINE_CHECK:
11892	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
11893	break;
11894
11895	default:
11896	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11897	break;
11898	}
11899	}
11900
11901	/*
11902	* Check for debugger event breakpoints and dtrace probes.
11903	*/
11904	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
11905	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
11906	{
11907	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
11908	if (rcStrict != VINF_SUCCESS)
11909	return rcStrict;
11910	}
11911
11912	/*
11913	* Normal processing.
11914	*/
11915	#ifdef HMVMX_USE_FUNCTION_TABLE
11916	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
11917	#else
11918	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
11919	#endif
11920	}
11921
11922
11923	/**
11924	* Single steps guest code using hardware-assisted VMX.
11925	*
11926	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
11927	* but single-stepping through the hypervisor debugger.
11928	*
11929	* @returns Strict VBox status code (i.e. informational status codes too).
11930	* @param pVCpu The cross context virtual CPU structure.
11931	* @param pcLoops Pointer to the number of executed loops.
11932	*
11933	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
11934	*/
11935	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
11936	{
11937	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11938	Assert(pcLoops);
11939	Assert(*pcLoops <= cMaxResumeLoops);
11940
11941	VMXTRANSIENT VmxTransient;
11942	RT_ZERO(VmxTransient);
11943	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11944
11945	/* Set HMCPU indicators. */
11946	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
11947	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
11948	pVCpu->hm.s.fDebugWantRdTscExit = false;
11949	pVCpu->hm.s.fUsingDebugLoop = true;
11950
11951	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
11952	VMXRUNDBGSTATE DbgState;
11953	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
11954	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
11955
11956	/*
11957	* The loop.
11958	*/
11959	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11960	for (;;)
11961	{
11962	Assert(!HMR0SuspendPending());
11963	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11964	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11965	bool fStepping = pVCpu->hm.s.fSingleInstruction;
11966
11967	/* Set up VM-execution controls the next two can respond to. */
11968	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
11969
11970	/*
11971	* Preparatory work for running guest code, this may force us to
11972	* return to ring-3.
11973	*
11974	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11975	*/
11976	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
11977	if (rcStrict != VINF_SUCCESS)
11978	break;
11979
11980	/* Interrupts are disabled at this point! */
11981	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11982
11983	/* Override any obnoxious code in the above two calls. */
11984	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
11985
11986	/*
11987	* Finally execute the guest.
11988	*/
11989	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11990
11991	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11992	/* Interrupts are re-enabled at this point! */
11993
11994	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
11995	if (RT_SUCCESS(rcRun))
11996	{ /* very likely */ }
11997	else
11998	{
11999	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12000	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12001	return rcRun;
12002	}
12003
12004	/* Profile the VM-exit. */
12005	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12006	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12007	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12008	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12009	HMVMX_START_EXIT_DISPATCH_PROF();
12010
12011	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12012
12013	/*
12014	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12015	*/
12016	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12017	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12018	if (rcStrict != VINF_SUCCESS)
12019	break;
12020	if (++(*pcLoops) > cMaxResumeLoops)
12021	{
12022	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12023	rcStrict = VINF_EM_RAW_INTERRUPT;
12024	break;
12025	}
12026
12027	/*
12028	* Stepping: Did the RIP change, if so, consider it a single step.
12029	* Otherwise, make sure one of the TFs gets set.
12030	*/
12031	if (fStepping)
12032	{
12033	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12034	AssertRC(rc);
12035	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12036	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12037	{
12038	rcStrict = VINF_EM_DBG_STEPPED;
12039	break;
12040	}
12041	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12042	}
12043
12044	/*
12045	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12046	*/
12047	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12048	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12049	}
12050
12051	/*
12052	* Clear the X86_EFL_TF if necessary.
12053	*/
12054	if (pVCpu->hm.s.fClearTrapFlag)
12055	{
12056	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12057	AssertRC(rc);
12058	pVCpu->hm.s.fClearTrapFlag = false;
12059	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12060	}
12061	/** @todo there seems to be issues with the resume flag when the monitor trap
12062	* flag is pending without being used. Seen early in bios init when
12063	* accessing APIC page in protected mode. */
12064
12065	/*
12066	* Restore VM-exit control settings as we may not re-enter this function the
12067	* next time around.
12068	*/
12069	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12070
12071	/* Restore HMCPU indicators. */
12072	pVCpu->hm.s.fUsingDebugLoop = false;
12073	pVCpu->hm.s.fDebugWantRdTscExit = false;
12074	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12075
12076	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12077	return rcStrict;
12078	}
12079
12080
12081	/** @} */
12082
12083
12084	/**
12085	* Checks if any expensive dtrace probes are enabled and we should go to the
12086	* debug loop.
12087	*
12088	* @returns true if we should use debug loop, false if not.
12089	*/
12090	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12091	{
12092	/* It's probably faster to OR the raw 32-bit counter variables together.
12093	Since the variables are in an array and the probes are next to one
12094	another (more or less), we have good locality. So, better read
12095	eight-nine cache lines ever time and only have one conditional, than
12096	128+ conditionals, right? */
12097	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12098	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12099	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12100	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12101	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12102	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12103	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12104	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12105	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12106	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12107	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12108	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12109	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12110	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12111	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12112	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12113	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12114	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12115	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12116	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12117	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12118	) != 0
12119	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12120	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12121	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12122	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12123	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12124	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12125	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12126	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12127	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12128	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12129	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12130	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12131	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12132	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12133	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12134	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12135	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12136	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12137	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12138	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12139	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12140	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12141	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12142	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12143	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12144	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12145	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12146	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12147	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12148	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12149	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12150	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12151	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12152	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12153	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12154	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12155	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12156	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12157	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12158	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12159	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12160	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12161	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12162	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12163	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12164	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12165	) != 0
12166	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12167	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12168	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12169	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12170	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12171	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12172	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12173	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12174	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12175	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12176	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12177	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12178	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12179	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12180	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12181	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12182	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12183	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12184	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12185	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12186	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12187	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12188	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12189	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12190	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12191	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12192	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12193	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12194	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12195	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12196	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12197	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12198	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12199	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12200	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12201	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12202	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12203	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12204	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12205	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12206	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12207	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12208	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12209	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12210	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12211	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12212	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12213	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12214	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12215	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12216	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12217	) != 0;
12218	}
12219
12220
12221	/**
12222	* Runs the guest using hardware-assisted VMX.
12223	*
12224	* @returns Strict VBox status code (i.e. informational status codes too).
12225	* @param pVCpu The cross context virtual CPU structure.
12226	*/
12227	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
12228	{
12229	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12230	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12231	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12232	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12233
12234	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
12235
12236	VBOXSTRICTRC rcStrict;
12237	uint32_t cLoops = 0;
12238	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12239	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12240	#else
12241	bool const fInNestedGuestMode = false;
12242	#endif
12243	if (!fInNestedGuestMode)
12244	{
12245	if ( !pVCpu->hm.s.fUseDebugLoop
12246	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12247	&& !DBGFIsStepping(pVCpu)
12248	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12249	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12250	else
12251	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12252	}
12253	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12254	else
12255	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
12256
12257	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12258	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12259	#endif
12260
12261	if (rcStrict == VERR_EM_INTERPRETER)
12262	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12263	else if (rcStrict == VINF_EM_RESET)
12264	rcStrict = VINF_EM_TRIPLE_FAULT;
12265
12266	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12267	if (RT_FAILURE(rc2))
12268	{
12269	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12270	rcStrict = rc2;
12271	}
12272	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12273	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12274	return rcStrict;
12275	}
12276
12277
12278	#ifndef HMVMX_USE_FUNCTION_TABLE
12279	/**
12280	* Handles a guest VM-exit from hardware-assisted VMX execution.
12281	*
12282	* @returns Strict VBox status code (i.e. informational status codes too).
12283	* @param pVCpu The cross context virtual CPU structure.
12284	* @param pVmxTransient The VMX-transient structure.
12285	*/
12286	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12287	{
12288	#ifdef DEBUG_ramshankar
12289	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12290	do { \
12291	if (a_fSave != 0) \
12292	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
12293	VBOXSTRICTRC rcStrict = a_CallExpr; \
12294	if (a_fSave != 0) \
12295	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12296	return rcStrict; \
12297	} while (0)
12298	#else
12299	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12300	#endif
12301	uint32_t const rcReason = pVmxTransient->uExitReason;
12302	switch (rcReason)
12303	{
12304	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12305	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12306	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12307	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12308	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12309	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12310	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12311	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12312	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12313	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12314	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12315	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12316	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12317	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12318	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12319	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12320	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12321	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12322	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12323	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12324	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12325	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12326	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12327	case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
12328	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12329	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12330	case VMX_EXIT_GDTR_IDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12331	case VMX_EXIT_LDTR_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12332	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12333	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12334	case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
12335	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12336	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12337	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12338	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12339	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12340	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12341	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12342	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12343	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12344	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12345	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12346	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12347	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12348	#else
12349	case VMX_EXIT_VMCLEAR:
12350	case VMX_EXIT_VMLAUNCH:
12351	case VMX_EXIT_VMPTRLD:
12352	case VMX_EXIT_VMPTRST:
12353	case VMX_EXIT_VMREAD:
12354	case VMX_EXIT_VMRESUME:
12355	case VMX_EXIT_VMWRITE:
12356	case VMX_EXIT_VMXOFF:
12357	case VMX_EXIT_VMXON:
12358	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12359	#endif
12360
12361	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12362	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12363	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
12364	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
12365	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
12366	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
12367	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
12368	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12369	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
12370
12371	case VMX_EXIT_INVEPT:
12372	case VMX_EXIT_INVVPID:
12373	case VMX_EXIT_VMFUNC:
12374	case VMX_EXIT_XSAVES:
12375	case VMX_EXIT_XRSTORS:
12376	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12377
12378	case VMX_EXIT_ENCLS:
12379	case VMX_EXIT_RDSEED:
12380	case VMX_EXIT_PML_FULL:
12381	default:
12382	return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
12383	}
12384	#undef VMEXIT_CALL_RET
12385	}
12386	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12387
12388
12389	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12390	/**
12391	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12392	*
12393	* @returns Strict VBox status code (i.e. informational status codes too).
12394	* @param pVCpu The cross context virtual CPU structure.
12395	* @param pVmxTransient The VMX-transient structure.
12396	*/
12397	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12398	{
12399	uint32_t const rcReason = pVmxTransient->uExitReason;
12400	switch (rcReason)
12401	{
12402	case VMX_EXIT_EPT_MISCONFIG:
12403	case VMX_EXIT_EPT_VIOLATION:
12404	case VMX_EXIT_IO_INSTR:
12405	case VMX_EXIT_CPUID:
12406	case VMX_EXIT_RDTSC:
12407	case VMX_EXIT_RDTSCP:
12408	case VMX_EXIT_APIC_ACCESS:
12409	case VMX_EXIT_XCPT_OR_NMI:
12410	case VMX_EXIT_MOV_CRX:
12411	case VMX_EXIT_EXT_INT:
12412	case VMX_EXIT_INT_WINDOW:
12413	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12414	case VMX_EXIT_MWAIT:
12415	case VMX_EXIT_MONITOR:
12416	case VMX_EXIT_TASK_SWITCH:
12417	case VMX_EXIT_PREEMPT_TIMER:
12418	case VMX_EXIT_RDMSR:
12419	case VMX_EXIT_WRMSR:
12420	case VMX_EXIT_VMCALL:
12421	case VMX_EXIT_MOV_DRX:
12422	case VMX_EXIT_HLT:
12423	case VMX_EXIT_INVD:
12424	case VMX_EXIT_INVLPG:
12425	case VMX_EXIT_RSM:
12426	case VMX_EXIT_MTF:
12427	case VMX_EXIT_PAUSE:
12428	case VMX_EXIT_GDTR_IDTR_ACCESS:
12429	case VMX_EXIT_LDTR_TR_ACCESS:
12430	case VMX_EXIT_WBINVD:
12431	case VMX_EXIT_XSETBV:
12432	case VMX_EXIT_RDRAND:
12433	case VMX_EXIT_INVPCID:
12434	case VMX_EXIT_GETSEC:
12435	case VMX_EXIT_RDPMC:
12436	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12437	case VMX_EXIT_VMCLEAR:
12438	case VMX_EXIT_VMLAUNCH:
12439	case VMX_EXIT_VMPTRLD:
12440	case VMX_EXIT_VMPTRST:
12441	case VMX_EXIT_VMREAD:
12442	case VMX_EXIT_VMRESUME:
12443	case VMX_EXIT_VMWRITE:
12444	case VMX_EXIT_VMXOFF:
12445	case VMX_EXIT_VMXON:
12446	#endif
12447	case VMX_EXIT_TRIPLE_FAULT:
12448	case VMX_EXIT_NMI_WINDOW:
12449	case VMX_EXIT_INIT_SIGNAL:
12450	case VMX_EXIT_SIPI:
12451	case VMX_EXIT_IO_SMI:
12452	case VMX_EXIT_SMI:
12453	case VMX_EXIT_ERR_MSR_LOAD:
12454	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12455	case VMX_EXIT_ERR_MACHINE_CHECK:
12456
12457	case VMX_EXIT_INVEPT:
12458	case VMX_EXIT_INVVPID:
12459	case VMX_EXIT_VMFUNC:
12460	case VMX_EXIT_XSAVES:
12461	case VMX_EXIT_XRSTORS:
12462
12463	case VMX_EXIT_ENCLS:
12464	case VMX_EXIT_RDSEED:
12465	case VMX_EXIT_PML_FULL:
12466	default:
12467	return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
12468	}
12469	#undef VMEXIT_CALL_RET
12470	}
12471	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12472
12473
12474	#ifdef VBOX_STRICT
12475	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
12476	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
12477	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
12478
12479	# define HMVMX_ASSERT_PREEMPT_CPUID() \
12480	do { \
12481	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
12482	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
12483	} while (0)
12484
12485	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12486	do { \
12487	AssertPtr((a_pVCpu)); \
12488	AssertPtr((a_pVmxTransient)); \
12489	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
12490	Assert((a_pVmxTransient)->pVmcsInfo); \
12491	Assert(ASMIntAreEnabled()); \
12492	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12493	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
12494	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
12495	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12496	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
12497	HMVMX_ASSERT_PREEMPT_CPUID(); \
12498	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12499	} while (0)
12500
12501	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12502	do { \
12503	Log4Func(("\n")); \
12504	} while (0)
12505	#else
12506	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12507	do { \
12508	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12509	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
12510	} while (0)
12511	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
12512	#endif
12513
12514
12515	/**
12516	* Advances the guest RIP by the specified number of bytes.
12517	*
12518	* @param pVCpu The cross context virtual CPU structure.
12519	* @param cbInstr Number of bytes to advance the RIP by.
12520	*
12521	* @remarks No-long-jump zone!!!
12522	*/
12523	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
12524	{
12525	/* Advance the RIP. */
12526	pVCpu->cpum.GstCtx.rip += cbInstr;
12527	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12528
12529	/* Update interrupt inhibition. */
12530	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
12531	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
12532	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12533	}
12534
12535
12536	/**
12537	* Advances the guest RIP after reading it from the VMCS.
12538	*
12539	* @returns VBox status code, no informational status codes.
12540	* @param pVCpu The cross context virtual CPU structure.
12541	* @param pVmxTransient The VMX-transient structure.
12542	*
12543	* @remarks No-long-jump zone!!!
12544	*/
12545	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12546	{
12547	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12548	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
12549	AssertRCReturn(rc, rc);
12550
12551	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
12552	return VINF_SUCCESS;
12553	}
12554
12555
12556	/**
12557	* Handle a condition that occurred while delivering an event through the guest
12558	* IDT.
12559	*
12560	* @returns Strict VBox status code (i.e. informational status codes too).
12561	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
12562	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
12563	* to continue execution of the guest which will delivery the \#DF.
12564	* @retval VINF_EM_RESET if we detected a triple-fault condition.
12565	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
12566	*
12567	* @param pVCpu The cross context virtual CPU structure.
12568	* @param pVmxTransient The VMX-transient structure.
12569	*
12570	* @remarks No-long-jump zone!!!
12571	*/
12572	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12573	{
12574	uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12575
12576	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12577	rc2 \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12578	AssertRCReturn(rc2, rc2);
12579
12580	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
12581	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12582	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12583	{
12584	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12585	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12586
12587	/*
12588	* If the event was a software interrupt (generated with INT n) or a software exception
12589	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
12590	* can handle the VM-exit and continue guest execution which will re-execute the
12591	* instruction rather than re-injecting the exception, as that can cause premature
12592	* trips to ring-3 before injection and involve TRPM which currently has no way of
12593	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
12594	* the problem).
12595	*/
12596	IEMXCPTRAISE enmRaise;
12597	IEMXCPTRAISEINFO fRaiseInfo;
12598	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
12599	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
12600	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
12601	{
12602	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
12603	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12604	}
12605	else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
12606	{
12607	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12608	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
12609	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
12610	/** @todo Make AssertMsgReturn as just AssertMsg later. */
12611	AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
12612	("Unexpected VM-exit interruption vector type %#x!\n", uExitVectorType), VERR_VMX_IPE_5);
12613
12614	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
12615
12616	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
12617	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
12618	{
12619	pVmxTransient->fVectoringPF = true;
12620	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12621	}
12622	}
12623	else
12624	{
12625	/*
12626	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
12627	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
12628	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
12629	*/
12630	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12631	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12632	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
12633	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12634	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12635	}
12636
12637	/*
12638	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
12639	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
12640	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
12641	* subsequent VM-entry would fail.
12642	*
12643	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
12644	*/
12645	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS)
12646	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12647	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
12648	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
12649	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
12650	{
12651	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
12652	}
12653
12654	switch (enmRaise)
12655	{
12656	case IEMXCPTRAISE_CURRENT_XCPT:
12657	{
12658	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
12659	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
12660	Assert(rcStrict == VINF_SUCCESS);
12661	break;
12662	}
12663
12664	case IEMXCPTRAISE_PREV_EVENT:
12665	{
12666	uint32_t u32ErrCode;
12667	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
12668	{
12669	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12670	AssertRCReturn(rc2, rc2);
12671	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12672	}
12673	else
12674	u32ErrCode = 0;
12675
12676	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
12677	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
12678	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12679	0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
12680
12681	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
12682	pVCpu->hm.s.Event.u32ErrCode));
12683	Assert(rcStrict == VINF_SUCCESS);
12684	break;
12685	}
12686
12687	case IEMXCPTRAISE_REEXEC_INSTR:
12688	Assert(rcStrict == VINF_SUCCESS);
12689	break;
12690
12691	case IEMXCPTRAISE_DOUBLE_FAULT:
12692	{
12693	/*
12694	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
12695	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
12696	*/
12697	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
12698	{
12699	pVmxTransient->fVectoringDoublePF = true;
12700	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
12701	pVCpu->cpum.GstCtx.cr2));
12702	rcStrict = VINF_SUCCESS;
12703	}
12704	else
12705	{
12706	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
12707	hmR0VmxSetPendingXcptDF(pVCpu);
12708	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
12709	uIdtVector, uExitVector));
12710	rcStrict = VINF_HM_DOUBLE_FAULT;
12711	}
12712	break;
12713	}
12714
12715	case IEMXCPTRAISE_TRIPLE_FAULT:
12716	{
12717	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
12718	rcStrict = VINF_EM_RESET;
12719	break;
12720	}
12721
12722	case IEMXCPTRAISE_CPU_HANG:
12723	{
12724	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
12725	rcStrict = VERR_EM_GUEST_CPU_HANG;
12726	break;
12727	}
12728
12729	default:
12730	{
12731	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
12732	rcStrict = VERR_VMX_IPE_2;
12733	break;
12734	}
12735	}
12736	}
12737	else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
12738	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
12739	&& uExitVector != X86_XCPT_DF
12740	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
12741	{
12742	/*
12743	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
12744	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
12745	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
12746	*/
12747	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
12748	{
12749	Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
12750	VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
12751	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
12752	}
12753	}
12754
12755	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
12756	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
12757	return rcStrict;
12758	}
12759
12760
12761	/** @name VM-exit handlers.
12762	* @{
12763	*/
12764	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12765	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12766	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12767
12768	/**
12769	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
12770	*/
12771	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12772	{
12773	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12774	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
12775	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
12776	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
12777	return VINF_SUCCESS;
12778	return VINF_EM_RAW_INTERRUPT;
12779	}
12780
12781
12782	/**
12783	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
12784	*/
12785	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12786	{
12787	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12788	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
12789
12790	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12791	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12792	AssertRCReturn(rc, rc);
12793
12794	uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12795	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
12796	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
12797	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
12798
12799	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12800	{
12801	/*
12802	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
12803	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
12804	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
12805	*
12806	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
12807	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
12808	*/
12809	VMXDispatchHostNmi();
12810	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
12811	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12812	return VINF_SUCCESS;
12813	}
12814
12815	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12816	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12817	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
12818	{ /* likely */ }
12819	else
12820	{
12821	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
12822	rcStrictRc1 = VINF_SUCCESS;
12823	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12824	return rcStrictRc1;
12825	}
12826
12827	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
12828	uint32_t uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
12829	switch (uIntType)
12830	{
12831	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
12832	Assert(uVector == X86_XCPT_DB);
12833	RT_FALL_THRU();
12834	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
12835	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
12836	RT_FALL_THRU();
12837	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
12838	{
12839	/*
12840	* If there's any exception caused as a result of event injection, the resulting
12841	* secondary/final execption will be pending, we shall continue guest execution
12842	* after injecting the event. The page-fault case is complicated and we manually
12843	* handle any currently pending event in hmR0VmxExitXcptPF.
12844	*/
12845	if (!pVCpu->hm.s.Event.fPending)
12846	{ /* likely */ }
12847	else if (uVector != X86_XCPT_PF)
12848	{
12849	rc = VINF_SUCCESS;
12850	break;
12851	}
12852
12853	switch (uVector)
12854	{
12855	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
12856	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
12857	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
12858	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
12859	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
12860	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
12861
12862	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
12863	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12864	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
12865	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12866	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
12867	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12868	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
12869	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12870	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
12871	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12872	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
12873	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12874	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
12875	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12876	default:
12877	{
12878	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
12879	if (pVmcsInfo->RealMode.fRealOnV86Active)
12880	{
12881	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
12882	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
12883	Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
12884
12885	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
12886	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12887	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12888	AssertRCReturn(rc, rc);
12889	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
12890	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
12891	0 /* GCPtrFaultAddress */);
12892	}
12893	else
12894	{
12895	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
12896	pVCpu->hm.s.u32HMError = uVector;
12897	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
12898	}
12899	break;
12900	}
12901	}
12902	break;
12903	}
12904
12905	default:
12906	{
12907	pVCpu->hm.s.u32HMError = uExitIntInfo;
12908	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
12909	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
12910	break;
12911	}
12912	}
12913	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12914	return rc;
12915	}
12916
12917
12918	/**
12919	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
12920	*/
12921	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12922	{
12923	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12924
12925	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
12926	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12927	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
12928	AssertRCReturn(rc, rc);
12929
12930	/* Evaluate and deliver pending events and resume guest execution. */
12931	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
12932	return VINF_SUCCESS;
12933	}
12934
12935
12936	/**
12937	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
12938	*/
12939	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12940	{
12941	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12942
12943	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12944	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
12945	{
12946	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
12947	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12948	}
12949
12950	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS));
12951
12952	/*
12953	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
12954	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
12955	*/
12956	uint32_t fIntrState;
12957	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
12958	AssertRCReturn(rc, rc);
12959	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
12960	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
12961	{
12962	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
12963	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12964
12965	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
12966	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
12967	AssertRCReturn(rc, rc);
12968	}
12969
12970	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
12971	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
12972	AssertRCReturn(rc, rc);
12973
12974	/* Evaluate and deliver pending events and resume guest execution. */
12975	return VINF_SUCCESS;
12976	}
12977
12978
12979	/**
12980	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
12981	*/
12982	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12983	{
12984	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12985	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12986	}
12987
12988
12989	/**
12990	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
12991	*/
12992	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12993	{
12994	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12995	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12996	}
12997
12998
12999	/**
13000	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
13001	*/
13002	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13003	{
13004	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13005
13006	/*
13007	* Get the state we need and update the exit history entry.
13008	*/
13009	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13010	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13011	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13012	AssertRCReturn(rc, rc);
13013
13014	VBOXSTRICTRC rcStrict;
13015	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
13016	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
13017	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
13018	if (!pExitRec)
13019	{
13020	/*
13021	* Regular CPUID instruction execution.
13022	*/
13023	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
13024	if (rcStrict == VINF_SUCCESS)
13025	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13026	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13027	{
13028	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13029	rcStrict = VINF_SUCCESS;
13030	}
13031	}
13032	else
13033	{
13034	/*
13035	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
13036	*/
13037	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13038	AssertRCReturn(rc2, rc2);
13039
13040	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
13041	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
13042
13043	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
13044	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13045
13046	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
13047	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13048	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
13049	}
13050	return rcStrict;
13051	}
13052
13053
13054	/**
13055	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
13056	*/
13057	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13058	{
13059	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13060
13061	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13062	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
13063	AssertRCReturn(rc, rc);
13064
13065	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
13066	return VINF_EM_RAW_EMULATE_INSTR;
13067
13068	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
13069	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13070	}
13071
13072
13073	/**
13074	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
13075	*/
13076	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13077	{
13078	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13079
13080	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13081	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13082	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13083	AssertRCReturn(rc, rc);
13084
13085	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
13086	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13087	{
13088	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13089	we must reset offsetting on VM-entry. See @bugref{6634}. */
13090	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13091	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13092	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13093	}
13094	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13095	{
13096	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13097	rcStrict = VINF_SUCCESS;
13098	}
13099	return rcStrict;
13100	}
13101
13102
13103	/**
13104	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
13105	*/
13106	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13107	{
13108	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13109
13110	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13111	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
13112	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13113	AssertRCReturn(rc, rc);
13114
13115	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
13116	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13117	{
13118	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13119	we must reset offsetting on VM-reentry. See @bugref{6634}. */
13120	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13121	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13122	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13123	}
13124	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13125	{
13126	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13127	rcStrict = VINF_SUCCESS;
13128	}
13129	return rcStrict;
13130	}
13131
13132
13133	/**
13134	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
13135	*/
13136	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13137	{
13138	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13139
13140	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13141	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
13142	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13143	AssertRCReturn(rc, rc);
13144
13145	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13146	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13147	if (RT_LIKELY(rc == VINF_SUCCESS))
13148	{
13149	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13150	Assert(pVmxTransient->cbInstr == 2);
13151	}
13152	else
13153	{
13154	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
13155	rc = VERR_EM_INTERPRETER;
13156	}
13157	return rc;
13158	}
13159
13160
13161	/**
13162	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
13163	*/
13164	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13165	{
13166	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13167
13168	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
13169	if (EMAreHypercallInstructionsEnabled(pVCpu))
13170	{
13171	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13172	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
13173	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13174	AssertRCReturn(rc, rc);
13175
13176	/* Perform the hypercall. */
13177	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
13178	if (rcStrict == VINF_SUCCESS)
13179	{
13180	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13181	AssertRCReturn(rc, rc);
13182	}
13183	else
13184	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
13185	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
13186	\|\| RT_FAILURE(rcStrict));
13187
13188	/* If the hypercall changes anything other than guest's general-purpose registers,
13189	we would need to reload the guest changed bits here before VM-entry. */
13190	}
13191	else
13192	Log4Func(("Hypercalls not enabled\n"));
13193
13194	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
13195	if (RT_FAILURE(rcStrict))
13196	{
13197	hmR0VmxSetPendingXcptUD(pVCpu);
13198	rcStrict = VINF_SUCCESS;
13199	}
13200
13201	return rcStrict;
13202	}
13203
13204
13205	/**
13206	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
13207	*/
13208	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13209	{
13210	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13211	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
13212
13213	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13214	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13215	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13216	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13217	AssertRCReturn(rc, rc);
13218
13219	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
13220
13221	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
13222	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13223	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13224	{
13225	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13226	rcStrict = VINF_SUCCESS;
13227	}
13228	else
13229	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n", pVmxTransient->uExitQual,
13230	VBOXSTRICTRC_VAL(rcStrict)));
13231	return rcStrict;
13232	}
13233
13234
13235	/**
13236	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
13237	*/
13238	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13239	{
13240	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13241
13242	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13243	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13244	AssertRCReturn(rc, rc);
13245
13246	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13247	rc = EMInterpretMonitor(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13248	if (RT_LIKELY(rc == VINF_SUCCESS))
13249	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13250	else
13251	{
13252	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
13253	rc = VERR_EM_INTERPRETER;
13254	}
13255	return rc;
13256	}
13257
13258
13259	/**
13260	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
13261	*/
13262	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13263	{
13264	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13265
13266	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13267	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13268	AssertRCReturn(rc, rc);
13269
13270	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13271	VBOXSTRICTRC rc2 = EMInterpretMWait(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13272	rc = VBOXSTRICTRC_VAL(rc2);
13273	if (RT_LIKELY( rc == VINF_SUCCESS
13274	\|\| rc == VINF_EM_HALT))
13275	{
13276	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13277	AssertRCReturn(rc3, rc3);
13278
13279	if ( rc == VINF_EM_HALT
13280	&& EMMonitorWaitShouldContinue(pVCpu, pCtx))
13281	rc = VINF_SUCCESS;
13282	}
13283	else
13284	{
13285	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
13286	rc = VERR_EM_INTERPRETER;
13287	}
13288	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
13289	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
13290	return rc;
13291	}
13292
13293
13294	/**
13295	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
13296	*/
13297	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13298	{
13299	/*
13300	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
13301	* mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
13302	* treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
13303	* VMX root operation. If we get here, something funny is going on.
13304	*
13305	* See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
13306	*/
13307	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13308	AssertMsgFailed(("Unexpected RSM VM-exit\n"));
13309	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13310	}
13311
13312
13313	/**
13314	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
13315	*/
13316	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13317	{
13318	/*
13319	* This can only happen if we support dual-monitor treatment of SMI, which can be activated
13320	* by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
13321	* this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
13322	* an SMI. If we get here, something funny is going on.
13323	*
13324	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
13325	* See Intel spec. 25.3 "Other Causes of VM-Exits"
13326	*/
13327	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13328	AssertMsgFailed(("Unexpected SMI VM-exit\n"));
13329	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13330	}
13331
13332
13333	/**
13334	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
13335	*/
13336	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13337	{
13338	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
13339	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13340	AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
13341	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13342	}
13343
13344
13345	/**
13346	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
13347	*/
13348	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13349	{
13350	/*
13351	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
13352	* We don't make use of it as our guests don't have direct access to the host LAPIC.
13353	* See Intel spec. 25.3 "Other Causes of VM-exits".
13354	*/
13355	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13356	AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
13357	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13358	}
13359
13360
13361	/**
13362	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
13363	* VM-exit.
13364	*/
13365	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13366	{
13367	/*
13368	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
13369	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
13370	*
13371	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
13372	* See Intel spec. "23.8 Restrictions on VMX operation".
13373	*/
13374	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13375	return VINF_SUCCESS;
13376	}
13377
13378
13379	/**
13380	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
13381	* VM-exit.
13382	*/
13383	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13384	{
13385	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13386	return VINF_EM_RESET;
13387	}
13388
13389
13390	/**
13391	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
13392	*/
13393	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13394	{
13395	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13396
13397	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13398	AssertRCReturn(rc, rc);
13399
13400	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
13401	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
13402	rc = VINF_SUCCESS;
13403	else
13404	rc = VINF_EM_HALT;
13405
13406	if (rc != VINF_SUCCESS)
13407	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
13408	return rc;
13409	}
13410
13411
13412	/**
13413	* VM-exit handler for instructions that result in a \#UD exception delivered to
13414	* the guest.
13415	*/
13416	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13417	{
13418	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13419	hmR0VmxSetPendingXcptUD(pVCpu);
13420	return VINF_SUCCESS;
13421	}
13422
13423
13424	/**
13425	* VM-exit handler for expiry of the VMX-preemption timer.
13426	*/
13427	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13428	{
13429	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13430
13431	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
13432	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13433
13434	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
13435	PVM pVM = pVCpu->CTX_SUFF(pVM);
13436	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
13437	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
13438	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
13439	}
13440
13441
13442	/**
13443	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
13444	*/
13445	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13446	{
13447	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13448
13449	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13450	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13451	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
13452	AssertRCReturn(rc, rc);
13453
13454	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
13455	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13456	: HM_CHANGED_RAISED_XCPT_MASK);
13457
13458	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13459	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
13460
13461	return rcStrict;
13462	}
13463
13464
13465	/**
13466	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
13467	*/
13468	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13469	{
13470	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13471	/** @todo Use VM-exit instruction information. */
13472	return VERR_EM_INTERPRETER;
13473	}
13474
13475
13476	/**
13477	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
13478	* Error VM-exit.
13479	*/
13480	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13481	{
13482	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13483	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13484	AssertRCReturn(rc, rc);
13485
13486	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo);
13487	if (RT_FAILURE(rc))
13488	return rc;
13489
13490	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
13491	NOREF(uInvalidReason);
13492
13493	#ifdef VBOX_STRICT
13494	uint32_t fIntrState;
13495	RTHCUINTREG uHCReg;
13496	uint64_t u64Val;
13497	uint32_t u32Val;
13498	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
13499	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
13500	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
13501	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13502	AssertRCReturn(rc, rc);
13503
13504	Log4(("uInvalidReason %u\n", uInvalidReason));
13505	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
13506	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
13507	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
13508	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
13509
13510	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
13511	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
13512	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
13513	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
13514	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
13515	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13516	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
13517	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
13518	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
13519	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13520	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
13521	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
13522
13523	hmR0DumpRegs(pVCpu);
13524	#endif
13525
13526	return VERR_VMX_INVALID_GUEST_STATE;
13527	}
13528
13529
13530	/**
13531	* VM-exit handler for VM-entry failure due to an MSR-load
13532	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
13533	*/
13534	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13535	{
13536	AssertMsgFailed(("Unexpected MSR-load exit\n"));
13537	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13538	}
13539
13540
13541	/**
13542	* VM-exit handler for VM-entry failure due to a machine-check event
13543	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
13544	*/
13545	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13546	{
13547	AssertMsgFailed(("Unexpected machine-check event exit\n"));
13548	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13549	}
13550
13551
13552	/**
13553	* VM-exit handler for all undefined reasons. Should never ever happen.. in
13554	* theory.
13555	*/
13556	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13557	{
13558	RT_NOREF2(pVCpu, pVmxTransient);
13559	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
13560	return VERR_VMX_UNDEFINED_EXIT_CODE;
13561	}
13562
13563
13564	/**
13565	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
13566	* (VMX_EXIT_GDTR_IDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
13567	* Conditional VM-exit.
13568	*/
13569	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13570	{
13571	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13572
13573	/* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
13574	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
13575	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13576	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
13577	return VERR_EM_INTERPRETER;
13578	AssertMsgFailed(("Unexpected XDTR access\n"));
13579	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13580	}
13581
13582
13583	/**
13584	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
13585	*/
13586	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13587	{
13588	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13589
13590	/* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
13591	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13592	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
13593	return VERR_EM_INTERPRETER;
13594	AssertMsgFailed(("Unexpected RDRAND exit\n"));
13595	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13596	}
13597
13598
13599	/**
13600	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
13601	*/
13602	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13603	{
13604	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13605
13606	/** @todo Optimize this: We currently drag in in the whole MSR state
13607	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13608	* MSRs required. That would require changes to IEM and possibly CPUM too.
13609	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13610	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13611	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13612	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13613	switch (idMsr)
13614	{
13615	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13616	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13617	}
13618
13619	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13620	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13621	AssertRCReturn(rc, rc);
13622
13623	Log4Func(("ecx=%#RX32\n", idMsr));
13624
13625	#ifdef VBOX_STRICT
13626	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
13627	{
13628	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
13629	&& idMsr != MSR_K6_EFER)
13630	{
13631	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
13632	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13633	}
13634	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13635	{
13636	Assert(pVmcsInfo->pvMsrBitmap);
13637	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13638	if (fMsrpm & VMXMSRPM_ALLOW_RD)
13639	{
13640	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
13641	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13642	}
13643	}
13644	}
13645	#endif
13646
13647	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
13648	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
13649	if (rcStrict == VINF_SUCCESS)
13650	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13651	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
13652	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13653	{
13654	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13655	rcStrict = VINF_SUCCESS;
13656	}
13657	else
13658	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13659
13660	return rcStrict;
13661	}
13662
13663
13664	/**
13665	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
13666	*/
13667	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13668	{
13669	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13670
13671	/** @todo Optimize this: We currently drag in in the whole MSR state
13672	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13673	* MSRs required. That would require changes to IEM and possibly CPUM too.
13674	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13675	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13676	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13677
13678	/*
13679	* The FS and GS base MSRs are not part of the above all-MSRs mask.
13680	* Although we don't need to fetch the base as it will be overwritten shortly, while
13681	* loading guest-state we would also load the entire segment register including limit
13682	* and attributes and thus we need to load them here.
13683	*/
13684	switch (idMsr)
13685	{
13686	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13687	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13688	}
13689
13690	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13691	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13692	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13693	AssertRCReturn(rc, rc);
13694
13695	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
13696
13697	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
13698	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
13699
13700	if (rcStrict == VINF_SUCCESS)
13701	{
13702	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13703
13704	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
13705	if ( idMsr == MSR_IA32_APICBASE
13706	\|\| ( idMsr >= MSR_IA32_X2APIC_START
13707	&& idMsr <= MSR_IA32_X2APIC_END))
13708	{
13709	/*
13710	* We've already saved the APIC related guest-state (TPR) in post-run phase.
13711	* When full APIC register virtualization is implemented we'll have to make
13712	* sure APIC state is saved from the VMCS before IEM changes it.
13713	*/
13714	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
13715	}
13716	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
13717	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13718	else if (idMsr == MSR_K6_EFER)
13719	{
13720	/*
13721	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
13722	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
13723	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
13724	*/
13725	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
13726	}
13727
13728	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
13729	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
13730	{
13731	switch (idMsr)
13732	{
13733	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
13734	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
13735	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
13736	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
13737	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
13738	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
13739	default:
13740	{
13741	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
13742	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
13743	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13744	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
13745	break;
13746	}
13747	}
13748	}
13749	#ifdef VBOX_STRICT
13750	else
13751	{
13752	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
13753	switch (idMsr)
13754	{
13755	case MSR_IA32_SYSENTER_CS:
13756	case MSR_IA32_SYSENTER_EIP:
13757	case MSR_IA32_SYSENTER_ESP:
13758	case MSR_K8_FS_BASE:
13759	case MSR_K8_GS_BASE:
13760	{
13761	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13762	Assert(fMsrpm == (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR));
13763
13764	uint32_t u32Proc;
13765	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Proc);
13766	AssertRC(rc);
13767	Assert(u32Proc == pVmcsInfo->u32ProcCtls);
13768	Assert(u32Proc & VMX_PROC_CTLS_USE_MSR_BITMAPS);
13769
13770	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32 fMsrpm=%#RX32\n", idMsr, fMsrpm));
13771	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13772	}
13773
13774	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
13775	default:
13776	{
13777	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
13778	{
13779	/* EFER MSR writes are always intercepted. */
13780	if (idMsr != MSR_K6_EFER)
13781	{
13782	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
13783	idMsr));
13784	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13785	}
13786	}
13787
13788	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13789	{
13790	Assert(pVmcsInfo->pvMsrBitmap);
13791	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13792	if (fMsrpm & VMXMSRPM_ALLOW_WR)
13793	{
13794	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
13795	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13796	}
13797	}
13798	break;
13799	}
13800	}
13801	}
13802	#endif /* VBOX_STRICT */
13803	}
13804	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13805	{
13806	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13807	rcStrict = VINF_SUCCESS;
13808	}
13809	else
13810	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13811
13812	return rcStrict;
13813	}
13814
13815
13816	/**
13817	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
13818	*/
13819	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13820	{
13821	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13822	/** @todo The guest has likely hit a contended spinlock. We might want to
13823	* poke a schedule different guest VCPU. */
13824	return VINF_EM_RAW_INTERRUPT;
13825	}
13826
13827
13828	/**
13829	* VM-exit handler for when the TPR value is lowered below the specified
13830	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
13831	*/
13832	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13833	{
13834	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13835	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
13836
13837	/*
13838	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
13839	* We'll re-evaluate pending interrupts and inject them before the next VM
13840	* entry so we can just continue execution here.
13841	*/
13842	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
13843	return VINF_SUCCESS;
13844	}
13845
13846
13847	/**
13848	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
13849	* VM-exit.
13850	*
13851	* @retval VINF_SUCCESS when guest execution can continue.
13852	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
13853	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
13854	* interpreter.
13855	*/
13856	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13857	{
13858	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13859	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
13860
13861	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13862	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13863	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13864	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13865	AssertRCReturn(rc, rc);
13866
13867	VBOXSTRICTRC rcStrict;
13868	PVM pVM = pVCpu->CTX_SUFF(pVM);
13869	RTGCUINTPTR const uExitQual = pVmxTransient->uExitQual;
13870	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
13871	switch (uAccessType)
13872	{
13873	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
13874	{
13875	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
13876	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
13877	VMX_EXIT_QUAL_CRX_GENREG(uExitQual));
13878	AssertMsg( rcStrict == VINF_SUCCESS
13879	\|\| rcStrict == VINF_IEM_RAISED_XCPT
13880	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13881
13882	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
13883	{
13884	case 0:
13885	{
13886	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13887	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13888	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
13889	Log4Func(("CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
13890
13891	/*
13892	* This is a kludge for handling switches back to real mode when we try to use
13893	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
13894	* deal with special selector values, so we have to return to ring-3 and run
13895	* there till the selector values are V86 mode compatible.
13896	*
13897	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
13898	* latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
13899	* at the end of this function.
13900	*/
13901	if ( rc == VINF_SUCCESS
13902	&& !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
13903	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
13904	&& (uOldCr0 & X86_CR0_PE)
13905	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
13906	{
13907	/** @todo check selectors rather than returning all the time. */
13908	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
13909	rcStrict = VINF_EM_RESCHEDULE_REM;
13910	}
13911	break;
13912	}
13913
13914	case 2:
13915	{
13916	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
13917	/* Nothing to do here, CR2 it's not part of the VMCS. */
13918	break;
13919	}
13920
13921	case 3:
13922	{
13923	Assert( !pVM->hm.s.fNestedPaging
13924	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
13925	\|\| pVCpu->hm.s.fUsingDebugLoop);
13926	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
13927	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13928	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
13929	Log4Func(("CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
13930	break;
13931	}
13932
13933	case 4:
13934	{
13935	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
13936	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13937	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
13938	Log4Func(("CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
13939	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
13940	break;
13941	}
13942
13943	case 8:
13944	{
13945	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
13946	Assert(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
13947	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13948	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
13949	break;
13950	}
13951	default:
13952	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual)));
13953	break;
13954	}
13955	break;
13956	}
13957
13958	case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
13959	{
13960	Assert( !pVM->hm.s.fNestedPaging
13961	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
13962	\|\| pVCpu->hm.s.fUsingDebugLoop
13963	\|\| VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 3);
13964	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
13965	Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 8
13966	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
13967
13968	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_GENREG(uExitQual),
13969	VMX_EXIT_QUAL_CRX_REGISTER(uExitQual));
13970	AssertMsg( rcStrict == VINF_SUCCESS
13971	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13972	#ifdef VBOX_WITH_STATISTICS
13973	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
13974	{
13975	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
13976	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
13977	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
13978	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
13979	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
13980	}
13981	#endif
13982	Log4Func(("CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
13983	VBOXSTRICTRC_VAL(rcStrict)));
13984	if (VMX_EXIT_QUAL_CRX_GENREG(uExitQual) == X86_GREG_xSP)
13985	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
13986	else
13987	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13988	break;
13989	}
13990
13991	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
13992	{
13993	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
13994	AssertMsg( rcStrict == VINF_SUCCESS
13995	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13996
13997	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13998	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
13999	Log4Func(("CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
14000	break;
14001	}
14002
14003	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
14004	{
14005	/* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
14006	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
14007	AssertRCReturn(rc, rc);
14008	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual),
14009	pVmxTransient->uGuestLinearAddr);
14010	AssertMsg( rcStrict == VINF_SUCCESS
14011	\|\| rcStrict == VINF_IEM_RAISED_XCPT
14012	, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14013
14014	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14015	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
14016	Log4Func(("LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
14017	break;
14018	}
14019
14020	default:
14021	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
14022	VERR_VMX_UNEXPECTED_EXCEPTION);
14023	}
14024
14025	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
14026	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
14027	if (rcStrict == VINF_IEM_RAISED_XCPT)
14028	{
14029	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14030	rcStrict = VINF_SUCCESS;
14031	}
14032
14033	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
14034	NOREF(pVM);
14035	return rcStrict;
14036	}
14037
14038
14039	/**
14040	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
14041	* VM-exit.
14042	*/
14043	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14044	{
14045	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14046	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
14047
14048	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14049	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14050	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14051	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14052	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
14053	\| CPUMCTX_EXTRN_EFER);
14054	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
14055	AssertRCReturn(rc, rc);
14056
14057	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
14058	uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
14059	uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQual);
14060	bool fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
14061	bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
14062	bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
14063	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
14064	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
14065
14066	/*
14067	* Update exit history to see if this exit can be optimized.
14068	*/
14069	VBOXSTRICTRC rcStrict;
14070	PCEMEXITREC pExitRec = NULL;
14071	if ( !fGstStepping
14072	&& !fDbgStepping)
14073	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14074	!fIOString
14075	? !fIOWrite
14076	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
14077	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
14078	: !fIOWrite
14079	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
14080	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
14081	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14082	if (!pExitRec)
14083	{
14084	/* I/O operation lookup arrays. */
14085	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
14086	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
14087	uint32_t const cbValue = s_aIOSizes[uIOWidth];
14088	uint32_t const cbInstr = pVmxTransient->cbInstr;
14089	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
14090	PVM pVM = pVCpu->CTX_SUFF(pVM);
14091	if (fIOString)
14092	{
14093	/*
14094	* INS/OUTS - I/O String instruction.
14095	*
14096	* Use instruction-information if available, otherwise fall back on
14097	* interpreting the instruction.
14098	*/
14099	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14100	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
14101	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
14102	if (fInsOutsInfo)
14103	{
14104	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14105	AssertRCReturn(rc2, rc2);
14106	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
14107	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
14108	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
14109	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
14110	if (fIOWrite)
14111	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
14112	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
14113	else
14114	{
14115	/*
14116	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
14117	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
14118	* See Intel Instruction spec. for "INS".
14119	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
14120	*/
14121	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
14122	}
14123	}
14124	else
14125	rcStrict = IEMExecOne(pVCpu);
14126
14127	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14128	fUpdateRipAlready = true;
14129	}
14130	else
14131	{
14132	/*
14133	* IN/OUT - I/O instruction.
14134	*/
14135	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14136	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
14137	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
14138	if (fIOWrite)
14139	{
14140	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
14141	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
14142	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14143	&& !pCtx->eflags.Bits.u1TF)
14144	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
14145	}
14146	else
14147	{
14148	uint32_t u32Result = 0;
14149	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
14150	if (IOM_SUCCESS(rcStrict))
14151	{
14152	/* Save result of I/O IN instr. in AL/AX/EAX. */
14153	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
14154	}
14155	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14156	&& !pCtx->eflags.Bits.u1TF)
14157	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
14158	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
14159	}
14160	}
14161
14162	if (IOM_SUCCESS(rcStrict))
14163	{
14164	if (!fUpdateRipAlready)
14165	{
14166	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
14167	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14168	}
14169
14170	/*
14171	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
14172	* while booting Fedora 17 64-bit guest.
14173	*
14174	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
14175	*/
14176	if (fIOString)
14177	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
14178
14179	/*
14180	* If any I/O breakpoints are armed, we need to check if one triggered
14181	* and take appropriate action.
14182	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
14183	*/
14184	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
14185	AssertRCReturn(rc, rc);
14186
14187	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
14188	* execution engines about whether hyper BPs and such are pending. */
14189	uint32_t const uDr7 = pCtx->dr[7];
14190	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
14191	&& X86_DR7_ANY_RW_IO(uDr7)
14192	&& (pCtx->cr4 & X86_CR4_DE))
14193	\|\| DBGFBpIsHwIoArmed(pVM)))
14194	{
14195	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
14196
14197	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
14198	VMMRZCallRing3Disable(pVCpu);
14199	HM_DISABLE_PREEMPT(pVCpu);
14200
14201	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
14202
14203	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
14204	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
14205	{
14206	/* Raise #DB. */
14207	if (fIsGuestDbgActive)
14208	ASMSetDR6(pCtx->dr[6]);
14209	if (pCtx->dr[7] != uDr7)
14210	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
14211
14212	hmR0VmxSetPendingXcptDB(pVCpu);
14213	}
14214	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
14215	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
14216	else if ( rcStrict2 != VINF_SUCCESS
14217	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
14218	rcStrict = rcStrict2;
14219	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
14220
14221	HM_RESTORE_PREEMPT();
14222	VMMRZCallRing3Enable(pVCpu);
14223	}
14224	}
14225
14226	#ifdef VBOX_STRICT
14227	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14228	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
14229	Assert(!fIOWrite);
14230	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14231	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
14232	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
14233	Assert(fIOWrite);
14234	else
14235	{
14236	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
14237	* statuses, that the VMM device and some others may return. See
14238	* IOM_SUCCESS() for guidance. */
14239	AssertMsg( RT_FAILURE(rcStrict)
14240	\|\| rcStrict == VINF_SUCCESS
14241	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
14242	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
14243	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
14244	\|\| rcStrict == VINF_EM_RAW_TO_R3
14245	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14246	# endif
14247	}
14248	#endif
14249	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
14250	}
14251	else
14252	{
14253	/*
14254	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14255	*/
14256	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14257	AssertRCReturn(rc2, rc2);
14258	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
14259	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
14260	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
14261	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14262	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
14263	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
14264
14265	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14266	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14267
14268	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14269	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14270	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14271	}
14272	return rcStrict;
14273	}
14274
14275
14276	/**
14277	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
14278	* VM-exit.
14279	*/
14280	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14281	{
14282	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14283
14284	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
14285	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14286	AssertRCReturn(rc, rc);
14287	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
14288	{
14289	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14290	AssertRCReturn(rc, rc);
14291	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
14292	{
14293	uint32_t uErrCode;
14294	RTGCUINTPTR GCPtrFaultAddress;
14295	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
14296	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
14297	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
14298	if (fErrorCodeValid)
14299	{
14300	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14301	AssertRCReturn(rc, rc);
14302	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
14303	}
14304	else
14305	uErrCode = 0;
14306
14307	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
14308	&& uVector == X86_XCPT_PF)
14309	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
14310	else
14311	GCPtrFaultAddress = 0;
14312
14313	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14314	AssertRCReturn(rc, rc);
14315
14316	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
14317	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
14318
14319	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
14320	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14321	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14322	}
14323	}
14324
14325	/* Fall back to the interpreter to emulate the task-switch. */
14326	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14327	return VERR_EM_INTERPRETER;
14328	}
14329
14330
14331	/**
14332	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
14333	*/
14334	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14335	{
14336	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14337
14338	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14339	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
14340	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14341	AssertRCReturn(rc, rc);
14342	return VINF_EM_DBG_STEPPED;
14343	}
14344
14345
14346	/**
14347	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
14348	*/
14349	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14350	{
14351	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14352	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
14353
14354	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14355	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14356	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14357	{
14358	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
14359	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14360	{
14361	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14362	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14363	}
14364	}
14365	else
14366	{
14367	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14368	rcStrict1 = VINF_SUCCESS;
14369	return rcStrict1;
14370	}
14371
14372	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
14373	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14374	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14375	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14376	AssertRCReturn(rc, rc);
14377
14378	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
14379	uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
14380	VBOXSTRICTRC rcStrict2;
14381	switch (uAccessType)
14382	{
14383	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
14384	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
14385	{
14386	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
14387	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
14388	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
14389
14390	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
14391	GCPhys &= PAGE_BASE_GC_MASK;
14392	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
14393	PVM pVM = pVCpu->CTX_SUFF(pVM);
14394	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
14395	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
14396
14397	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14398	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
14399	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
14400	CPUMCTX2CORE(pCtx), GCPhys);
14401	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14402	if ( rcStrict2 == VINF_SUCCESS
14403	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14404	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14405	{
14406	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14407	\| HM_CHANGED_GUEST_APIC_TPR);
14408	rcStrict2 = VINF_SUCCESS;
14409	}
14410	break;
14411	}
14412
14413	default:
14414	Log4Func(("uAccessType=%#x\n", uAccessType));
14415	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
14416	break;
14417	}
14418
14419	if (rcStrict2 != VINF_SUCCESS)
14420	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
14421	return rcStrict2;
14422	}
14423
14424
14425	/**
14426	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
14427	* VM-exit.
14428	*/
14429	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14430	{
14431	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14432
14433	/* We should -not- get this VM-exit if the guest's debug registers were active. */
14434	if (pVmxTransient->fWasGuestDebugStateActive)
14435	{
14436	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
14437	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14438	}
14439
14440	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14441	if ( !pVCpu->hm.s.fSingleInstruction
14442	&& !pVmxTransient->fWasHyperDebugStateActive)
14443	{
14444	Assert(!DBGFIsStepping(pVCpu));
14445	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
14446
14447	/* Don't intercept MOV DRx any more. */
14448	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
14449	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14450	AssertRCReturn(rc, rc);
14451
14452	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
14453	VMMRZCallRing3Disable(pVCpu);
14454	HM_DISABLE_PREEMPT(pVCpu);
14455
14456	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
14457	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
14458	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
14459
14460	HM_RESTORE_PREEMPT();
14461	VMMRZCallRing3Enable(pVCpu);
14462
14463	#ifdef VBOX_WITH_STATISTICS
14464	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14465	AssertRCReturn(rc, rc);
14466	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14467	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14468	else
14469	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14470	#endif
14471	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
14472	return VINF_SUCCESS;
14473	}
14474
14475	/*
14476	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
14477	* The EFER MSR is always up-to-date.
14478	* Update the segment registers and DR7 from the CPU.
14479	*/
14480	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14481	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14482	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
14483	AssertRCReturn(rc, rc);
14484	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
14485
14486	PVM pVM = pVCpu->CTX_SUFF(pVM);
14487	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14488	{
14489	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14490	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
14491	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
14492	if (RT_SUCCESS(rc))
14493	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
14494	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14495	}
14496	else
14497	{
14498	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14499	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
14500	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
14501	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14502	}
14503
14504	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
14505	if (RT_SUCCESS(rc))
14506	{
14507	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14508	AssertRCReturn(rc2, rc2);
14509	return VINF_SUCCESS;
14510	}
14511	return rc;
14512	}
14513
14514
14515	/**
14516	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
14517	* Conditional VM-exit.
14518	*/
14519	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14520	{
14521	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14522	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14523
14524	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14525	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14526	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14527	{
14528	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
14529	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
14530	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14531	{
14532	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14533	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14534	}
14535	}
14536	else
14537	{
14538	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14539	rcStrict1 = VINF_SUCCESS;
14540	return rcStrict1;
14541	}
14542
14543	/*
14544	* Get sufficent state and update the exit history entry.
14545	*/
14546	RTGCPHYS GCPhys;
14547	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14548	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14549	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14550	AssertRCReturn(rc, rc);
14551
14552	VBOXSTRICTRC rcStrict;
14553	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14554	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
14555	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14556	if (!pExitRec)
14557	{
14558	/*
14559	* If we succeed, resume guest execution.
14560	* If we fail in interpreting the instruction because we couldn't get the guest physical address
14561	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
14562	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
14563	* weird case. See @bugref{6043}.
14564	*/
14565	PVM pVM = pVCpu->CTX_SUFF(pVM);
14566	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14567	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
14568	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
14569	if ( rcStrict == VINF_SUCCESS
14570	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
14571	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
14572	{
14573	/* Successfully handled MMIO operation. */
14574	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14575	\| HM_CHANGED_GUEST_APIC_TPR);
14576	rcStrict = VINF_SUCCESS;
14577	}
14578	}
14579	else
14580	{
14581	/*
14582	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14583	*/
14584	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14585	AssertRCReturn(rc2, rc2);
14586
14587	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
14588	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
14589
14590	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14591	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14592
14593	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14594	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14595	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14596	}
14597	return VBOXSTRICTRC_TODO(rcStrict);
14598	}
14599
14600
14601	/**
14602	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
14603	* VM-exit.
14604	*/
14605	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14606	{
14607	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14608	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14609
14610	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14611	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14612	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14613	{
14614	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
14615	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14616	Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
14617	}
14618	else
14619	{
14620	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14621	rcStrict1 = VINF_SUCCESS;
14622	return rcStrict1;
14623	}
14624
14625	RTGCPHYS GCPhys;
14626	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14627	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14628	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14629	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14630	AssertRCReturn(rc, rc);
14631
14632	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
14633	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
14634
14635	RTGCUINT uErrorCode = 0;
14636	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
14637	uErrorCode \|= X86_TRAP_PF_ID;
14638	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
14639	uErrorCode \|= X86_TRAP_PF_RW;
14640	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
14641	uErrorCode \|= X86_TRAP_PF_P;
14642
14643	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
14644
14645
14646	/* Handle the pagefault trap for the nested shadow table. */
14647	PVM pVM = pVCpu->CTX_SUFF(pVM);
14648	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14649
14650	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
14651	pCtx->cs.Sel, pCtx->rip));
14652
14653	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
14654	TRPMResetTrap(pVCpu);
14655
14656	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
14657	if ( rcStrict2 == VINF_SUCCESS
14658	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14659	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14660	{
14661	/* Successfully synced our nested page tables. */
14662	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
14663	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
14664	return VINF_SUCCESS;
14665	}
14666
14667	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14668	return rcStrict2;
14669	}
14670
14671	/** @} */
14672
14673	/** @name VM-exit exception handlers.
14674	* @{
14675	*/
14676	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14677	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14678	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14679
14680	/**
14681	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
14682	*/
14683	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14684	{
14685	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14686	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
14687
14688	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
14689	AssertRCReturn(rc, rc);
14690
14691	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
14692	{
14693	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
14694	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
14695
14696	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
14697	* provides VM-exit instruction length. If this causes problem later,
14698	* disassemble the instruction like it's done on AMD-V. */
14699	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14700	AssertRCReturn(rc2, rc2);
14701	return rc;
14702	}
14703
14704	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14705	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14706	return rc;
14707	}
14708
14709
14710	/**
14711	* VM-exit exception handler for \#BP (Breakpoint exception).
14712	*/
14713	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14714	{
14715	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14716	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
14717
14718	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14719	AssertRCReturn(rc, rc);
14720
14721	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14722	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14723	if (rc == VINF_EM_RAW_GUEST_TRAP)
14724	{
14725	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14726	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14727	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14728	AssertRCReturn(rc, rc);
14729
14730	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14731	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14732	}
14733
14734	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
14735	return rc;
14736	}
14737
14738
14739	/**
14740	* VM-exit exception handler for \#AC (alignment check exception).
14741	*/
14742	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14743	{
14744	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14745
14746	/*
14747	* Re-inject it. We'll detect any nesting before getting here.
14748	*/
14749	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14750	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14751	AssertRCReturn(rc, rc);
14752	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
14753
14754	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14755	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14756	return VINF_SUCCESS;
14757	}
14758
14759
14760	/**
14761	* VM-exit exception handler for \#DB (Debug exception).
14762	*/
14763	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14764	{
14765	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14766	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
14767
14768	/*
14769	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
14770	* for processing.
14771	*/
14772	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14773
14774	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
14775	uint64_t uDR6 = X86_DR6_INIT_VAL;
14776	uDR6 \|= (pVmxTransient->uExitQual & (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
14777
14778	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14779	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
14780	Log6Func(("rc=%Rrc\n", rc));
14781	if (rc == VINF_EM_RAW_GUEST_TRAP)
14782	{
14783	/*
14784	* The exception was for the guest. Update DR6, DR7.GD and
14785	* IA32_DEBUGCTL.LBR before forwarding it.
14786	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
14787	*/
14788	VMMRZCallRing3Disable(pVCpu);
14789	HM_DISABLE_PREEMPT(pVCpu);
14790
14791	pCtx->dr[6] &= ~X86_DR6_B_MASK;
14792	pCtx->dr[6] \|= uDR6;
14793	if (CPUMIsGuestDebugStateActive(pVCpu))
14794	ASMSetDR6(pCtx->dr[6]);
14795
14796	HM_RESTORE_PREEMPT();
14797	VMMRZCallRing3Enable(pVCpu);
14798
14799	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
14800	AssertRCReturn(rc, rc);
14801
14802	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
14803	pCtx->dr[7] &= ~X86_DR7_GD;
14804
14805	/* Paranoia. */
14806	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
14807	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
14808
14809	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
14810	AssertRCReturn(rc, rc);
14811
14812	/*
14813	* Raise #DB in the guest.
14814	*
14815	* It is important to reflect exactly what the VM-exit gave us (preserving the
14816	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
14817	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
14818	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
14819	*
14820	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
14821	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
14822	*/
14823	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14824	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14825	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14826	AssertRCReturn(rc, rc);
14827	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14828	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14829	return VINF_SUCCESS;
14830	}
14831
14832	/*
14833	* Not a guest trap, must be a hypervisor related debug event then.
14834	* Update DR6 in case someone is interested in it.
14835	*/
14836	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
14837	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
14838	CPUMSetHyperDR6(pVCpu, uDR6);
14839
14840	return rc;
14841	}
14842
14843
14844	/**
14845	* Hacks its way around the lovely mesa driver's backdoor accesses.
14846	*
14847	* @sa hmR0SvmHandleMesaDrvGp
14848	*/
14849	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14850	{
14851	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
14852	RT_NOREF(pCtx);
14853
14854	/* For now we'll just skip the instruction. */
14855	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14856	}
14857
14858
14859	/**
14860	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
14861	* backdoor logging w/o checking what it is running inside.
14862	*
14863	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
14864	* backdoor port and magic numbers loaded in registers.
14865	*
14866	* @returns true if it is, false if it isn't.
14867	* @sa hmR0SvmIsMesaDrvGp
14868	*/
14869	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14870	{
14871	/* 0xed: IN eAX,dx */
14872	uint8_t abInstr[1];
14873	if (pVmxTransient->cbInstr != sizeof(abInstr))
14874	return false;
14875
14876	/* Check that it is #GP(0). */
14877	if (pVmxTransient->uExitIntErrorCode != 0)
14878	return false;
14879
14880	/* Check magic and port. */
14881	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
14882	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
14883	if (pCtx->rax != UINT32_C(0x564d5868))
14884	return false;
14885	if (pCtx->dx != UINT32_C(0x5658))
14886	return false;
14887
14888	/* Flat ring-3 CS. */
14889	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
14890	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
14891	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
14892	if (pCtx->cs.Attr.n.u2Dpl != 3)
14893	return false;
14894	if (pCtx->cs.u64Base != 0)
14895	return false;
14896
14897	/* Check opcode. */
14898	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
14899	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
14900	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
14901	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
14902	if (RT_FAILURE(rc))
14903	return false;
14904	if (abInstr[0] != 0xed)
14905	return false;
14906
14907	return true;
14908	}
14909
14910
14911	/**
14912	* VM-exit exception handler for \#GP (General-protection exception).
14913	*
14914	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
14915	*/
14916	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14917	{
14918	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14919	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
14920
14921	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14922	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14923	if (pVmcsInfo->RealMode.fRealOnV86Active)
14924	{ /* likely */ }
14925	else
14926	{
14927	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14928	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv);
14929	#endif
14930	/* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
14931	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14932	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14933	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14934	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14935	AssertRCReturn(rc, rc);
14936	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
14937	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
14938
14939	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
14940	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
14941	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14942	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14943	else
14944	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
14945	return rc;
14946	}
14947
14948	Assert(CPUMIsGuestInRealModeEx(pCtx));
14949	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
14950
14951	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14952	AssertRCReturn(rc, rc);
14953
14954	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
14955	if (rcStrict == VINF_SUCCESS)
14956	{
14957	if (!CPUMIsGuestInRealModeEx(pCtx))
14958	{
14959	/*
14960	* The guest is no longer in real-mode, check if we can continue executing the
14961	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
14962	*/
14963	pVmcsInfo->RealMode.fRealOnV86Active = false;
14964	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
14965	{
14966	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
14967	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14968	}
14969	else
14970	{
14971	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
14972	rcStrict = VINF_EM_RESCHEDULE;
14973	}
14974	}
14975	else
14976	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14977	}
14978	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14979	{
14980	rcStrict = VINF_SUCCESS;
14981	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14982	}
14983	return VBOXSTRICTRC_VAL(rcStrict);
14984	}
14985
14986
14987	/**
14988	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
14989	* the exception reported in the VMX transient structure back into the VM.
14990	*
14991	* @remarks Requires uExitIntInfo in the VMX transient structure to be
14992	* up-to-date.
14993	*/
14994	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14995	{
14996	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14997	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14998	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14999	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active,
15000	("uVector=%#x u32XcptBitmap=%#X32\n",
15001	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
15002	NOREF(pVmcsInfo);
15003	#endif
15004
15005	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
15006	hmR0VmxCheckExitDueToEventDelivery(). */
15007	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15008	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15009	AssertRCReturn(rc, rc);
15010	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15011
15012	#ifdef DEBUG_ramshankar
15013	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
15014	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n",
15015	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pCtx->cs.Sel, pCtx->rip));
15016	#endif
15017
15018	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15019	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15020	return VINF_SUCCESS;
15021	}
15022
15023
15024	/**
15025	* VM-exit exception handler for \#PF (Page-fault exception).
15026	*/
15027	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15028	{
15029	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15030	PVM pVM = pVCpu->CTX_SUFF(pVM);
15031	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15032	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15033	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15034	AssertRCReturn(rc, rc);
15035
15036	if (!pVM->hm.s.fNestedPaging)
15037	{ /* likely */ }
15038	else
15039	{
15040	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
15041	Assert(pVCpu->hm.s.fUsingDebugLoop);
15042	#endif
15043	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
15044	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
15045	{
15046	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15047	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
15048	}
15049	else
15050	{
15051	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15052	hmR0VmxSetPendingXcptDF(pVCpu);
15053	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
15054	}
15055	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15056	return rc;
15057	}
15058
15059	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
15060	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
15061	if (pVmxTransient->fVectoringPF)
15062	{
15063	Assert(pVCpu->hm.s.Event.fPending);
15064	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15065	}
15066
15067	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15068	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15069	AssertRCReturn(rc, rc);
15070
15071	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
15072	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
15073
15074	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
15075	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
15076
15077	Log4Func(("#PF: rc=%Rrc\n", rc));
15078	if (rc == VINF_SUCCESS)
15079	{
15080	/*
15081	* This is typically a shadow page table sync or a MMIO instruction. But we may have
15082	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
15083	*/
15084	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15085	TRPMResetTrap(pVCpu);
15086	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
15087	return rc;
15088	}
15089
15090	if (rc == VINF_EM_RAW_GUEST_TRAP)
15091	{
15092	if (!pVmxTransient->fVectoringDoublePF)
15093	{
15094	/* It's a guest page fault and needs to be reflected to the guest. */
15095	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
15096	TRPMResetTrap(pVCpu);
15097	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
15098	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15099	uGstErrorCode, pVmxTransient->uExitQual);
15100	}
15101	else
15102	{
15103	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15104	TRPMResetTrap(pVCpu);
15105	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
15106	hmR0VmxSetPendingXcptDF(pVCpu);
15107	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
15108	}
15109
15110	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15111	return VINF_SUCCESS;
15112	}
15113
15114	TRPMResetTrap(pVCpu);
15115	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
15116	return rc;
15117	}
15118
15119	/** @} */
15120
15121	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15122	/** @name VMX instruction handlers.
15123	* @{
15124	*/
15125	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15126	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VMX instructions VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15127	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15128
15129	/**
15130	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
15131	*/
15132	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15133	{
15134	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15135
15136	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15137	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15138	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15139	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15140	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15141	AssertRCReturn(rc, rc);
15142
15143	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15144
15145	VMXVEXITINFO ExitInfo;
15146	RT_ZERO(ExitInfo);
15147	ExitInfo.uReason = pVmxTransient->uExitReason;
15148	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15149	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15150	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15151	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15152
15153	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
15154	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15155	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15156	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15157	{
15158	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15159	rcStrict = VINF_SUCCESS;
15160	}
15161	return rcStrict;
15162	}
15163
15164
15165	/**
15166	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
15167	*/
15168	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15169	{
15170	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15171
15172	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
15173	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15174	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15175	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15176	AssertRCReturn(rc, rc);
15177
15178	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15179
15180	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
15181	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15182	{
15183	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15184	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15185	}
15186	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15187	return rcStrict;
15188	}
15189
15190
15191	/**
15192	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
15193	*/
15194	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15195	{
15196	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15197
15198	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15199	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15200	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15201	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15202	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15203	AssertRCReturn(rc, rc);
15204
15205	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15206
15207	VMXVEXITINFO ExitInfo;
15208	RT_ZERO(ExitInfo);
15209	ExitInfo.uReason = pVmxTransient->uExitReason;
15210	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15211	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15212	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15213	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15214
15215	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
15216	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15217	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15218	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15219	{
15220	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15221	rcStrict = VINF_SUCCESS;
15222	}
15223	return rcStrict;
15224	}
15225
15226
15227	/**
15228	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
15229	*/
15230	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15231	{
15232	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15233
15234	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15235	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15236	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15237	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15238	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15239	AssertRCReturn(rc, rc);
15240
15241	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15242
15243	VMXVEXITINFO ExitInfo;
15244	RT_ZERO(ExitInfo);
15245	ExitInfo.uReason = pVmxTransient->uExitReason;
15246	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15247	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15248	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15249	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15250
15251	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
15252	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15253	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15254	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15255	{
15256	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15257	rcStrict = VINF_SUCCESS;
15258	}
15259	return rcStrict;
15260	}
15261
15262
15263	/**
15264	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Unconditional VM-exit.
15265	*/
15266	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15267	{
15268	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15269
15270	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15271	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15272	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15273	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15274	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15275	AssertRCReturn(rc, rc);
15276
15277	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15278
15279	VMXVEXITINFO ExitInfo;
15280	RT_ZERO(ExitInfo);
15281	ExitInfo.uReason = pVmxTransient->uExitReason;
15282	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15283	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15284	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15285	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15286	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15287
15288	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
15289	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15290	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15291	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15292	{
15293	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15294	rcStrict = VINF_SUCCESS;
15295	}
15296	return rcStrict;
15297	}
15298
15299
15300	/**
15301	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
15302	*/
15303	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15304	{
15305	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15306
15307	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
15308	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15309	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15310	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15311	AssertRCReturn(rc, rc);
15312
15313	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15314
15315	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
15316	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15317	{
15318	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15319	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15320	}
15321	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15322	return rcStrict;
15323	}
15324
15325
15326	/**
15327	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Unconditional VM-exit.
15328	*/
15329	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15330	{
15331	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15332
15333	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15334	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15335	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15336	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15337	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15338	AssertRCReturn(rc, rc);
15339
15340	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15341
15342	VMXVEXITINFO ExitInfo;
15343	RT_ZERO(ExitInfo);
15344	ExitInfo.uReason = pVmxTransient->uExitReason;
15345	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15346	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15347	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15348	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15349	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15350
15351	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
15352	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15353	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15354	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15355	{
15356	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15357	rcStrict = VINF_SUCCESS;
15358	}
15359	return rcStrict;
15360	}
15361
15362
15363	/**
15364	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
15365	*/
15366	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15367	{
15368	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15369
15370	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15371	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
15372	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15373	AssertRCReturn(rc, rc);
15374
15375	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15376
15377	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
15378	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15379	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
15380	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15381	{
15382	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15383	rcStrict = VINF_SUCCESS;
15384	}
15385	return rcStrict;
15386	}
15387
15388
15389	/**
15390	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
15391	*/
15392	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15393	{
15394	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15395
15396	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15397	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15398	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15399	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15400	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15401	AssertRCReturn(rc, rc);
15402
15403	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15404
15405	VMXVEXITINFO ExitInfo;
15406	RT_ZERO(ExitInfo);
15407	ExitInfo.uReason = pVmxTransient->uExitReason;
15408	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15409	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15410	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15411	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15412
15413	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
15414	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15415	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15416	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15417	{
15418	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15419	rcStrict = VINF_SUCCESS;
15420	}
15421	return rcStrict;
15422	}
15423
15424	/** @} */
15425	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
15426

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 78221

Download in other formats: