HMVMXR0.cpp@ 74603

Last change on this file since 74603 was 74603, checked in by vboxsync, 6 years ago
VMM/IEM, HM: Nested VMX: bugref:9180 VM-exit bits; LMSW intercept. Separated VINF_HM_INTERCEPT_NOT_ACTIVE into VMX and SVM specific codes. Adjusted IEMExecDecodedLmsw to supply the additional memory operand parameter from the VMCS guest-linear address field.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 575.0 KB

Line
1	/* $Id: HMVMXR0.cpp 74603 2018-10-04 06:07:20Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2017 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27
28	#include <VBox/vmm/pdmapi.h>
29	#include <VBox/vmm/dbgf.h>
30	#include <VBox/vmm/iem.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#include <VBox/vmm/em.h>
35	#include <VBox/vmm/gim.h>
36	#include <VBox/vmm/apic.h>
37	#ifdef VBOX_WITH_REM
38	# include <VBox/vmm/rem.h>
39	#endif
40	#include "HMInternal.h"
41	#include <VBox/vmm/vm.h>
42	#include "HMVMXR0.h"
43	#include "dtrace/VBoxVMM.h"
44
45	#ifdef DEBUG_ramshankar
46	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
47	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
50	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
51	# define HMVMX_ALWAYS_TRAP_PF
52	# define HMVMX_ALWAYS_FLUSH_TLB
53	# define HMVMX_ALWAYS_SWAP_EFER
54	#endif
55
56
57	/*********************************************************************************************************************************
58	* Defined Constants And Macros *
59	*********************************************************************************************************************************/
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** Determine which tagged-TLB flush handler to use. */
64	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
66	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
67	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
68
69	/** @name HMVMX_READ_XXX
70	* Flags to skip redundant reads of some common VMCS fields that are not part of
71	* the guest-CPU or VCPU state but are needed while handling VM-exits.
72	*/
73	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
74	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
75	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
76	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
77	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
78	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
79	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
80	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
81	/** @} */
82
83	/**
84	* States of the VMCS.
85	*
86	* This does not reflect all possible VMCS states but currently only those
87	* needed for maintaining the VMCS consistently even when thread-context hooks
88	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
89	*/
90	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
91	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
92	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
93
94	/**
95	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
96	* guest using hardware-assisted VMX.
97	*
98	* This excludes state like GPRs (other than RSP) which are always are
99	* swapped and restored across the world-switch and also registers like EFER,
100	* MSR which cannot be modified by the guest without causing a VM-exit.
101	*/
102	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
103	\| CPUMCTX_EXTRN_RFLAGS \
104	\| CPUMCTX_EXTRN_RSP \
105	\| CPUMCTX_EXTRN_SREG_MASK \
106	\| CPUMCTX_EXTRN_TABLE_MASK \
107	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
108	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
109	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
110	\| CPUMCTX_EXTRN_TSC_AUX \
111	\| CPUMCTX_EXTRN_OTHER_MSRS \
112	\| CPUMCTX_EXTRN_CR0 \
113	\| CPUMCTX_EXTRN_CR3 \
114	\| CPUMCTX_EXTRN_CR4 \
115	\| CPUMCTX_EXTRN_DR7 \
116	\| CPUMCTX_EXTRN_HM_VMX_MASK)
117
118	/**
119	* Exception bitmap mask for real-mode guests (real-on-v86).
120	*
121	* We need to intercept all exceptions manually except:
122	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
123	* due to bugs in Intel CPUs.
124	* - \#PF need not be intercepted even in real-mode if we have Nested Paging
125	* support.
126	*/
127	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
128	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
129	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
130	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
131	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
132	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
133	\| RT_BIT(X86_XCPT_XF))
134
135	/** Maximum VM-instruction error number. */
136	#define HMVMX_INSTR_ERROR_MAX 28
137
138	/** Profiling macro. */
139	#ifdef HM_PROFILE_EXIT_DISPATCH
140	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
141	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
142	#else
143	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
144	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
145	#endif
146
147	/** Assert that preemption is disabled or covered by thread-context hooks. */
148	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
149	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
150
151	/** Assert that we haven't migrated CPUs when thread-context hooks are not
152	* used. */
153	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
154	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
155	("Illegal migration! Entered on CPU %u Current %u\n", \
156	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
157
158	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
159	* context. */
160	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
161	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
162	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
163
164	/** Macro for importing guest state from the VMCS back into CPUMCTX (intended to be
165	* used only from VM-exit handlers). */
166	#define HMVMX_CPUMCTX_IMPORT_STATE(a_pVCpu, a_fWhat) (hmR0VmxImportGuestState((a_pVCpu), (a_fWhat)))
167
168	/** Helper macro for VM-exit handlers called unexpectedly. */
169	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
170	do { \
171	(a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
172	return VERR_VMX_UNEXPECTED_EXIT; \
173	} while (0)
174
175	/** Macro for importing segment registers to the VMCS from the guest-CPU context. */
176	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
177	# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
178	hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
179	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
180	#else
181	# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
182	hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
183	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
184	#endif
185
186	/** Macro for exporting segment registers to the VMCS from the guest-CPU context. */
187	#define HMVMX_EXPORT_SREG(Sel, a_pCtxSelReg) \
188	hmR0VmxExportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
189	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
190
191	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
192	/** Macro that does the necessary privilege checks and intercepted VM-exits for
193	* guests that attempted to execute a VMX instruction. */
194	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
195	do \
196	{ \
197	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
198	if (rcStrictTmp == VINF_SUCCESS) \
199	{ /* likely */ } \
200	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
201	{ \
202	Assert((a_pVCpu)->hm.s.Event.fPending); \
203	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
204	return VINF_SUCCESS; \
205	} \
206	else \
207	{ \
208	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
209	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
210	} \
211	} while (0)
212
213	/** Macro that decodes a memory operand for an instruction VM-exit. */
214	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
215	do \
216	{ \
217	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
218	(a_pGCPtrEffAddr)); \
219	if (rcStrictTmp == VINF_SUCCESS) \
220	{ /* likely */ } \
221	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
222	{ \
223	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
224	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
225	return VINF_SUCCESS; \
226	} \
227	else \
228	{ \
229	Log4Func(("hmR0VmxCheckExitDueToVmxInstr failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
230	return rcStrictTmp; \
231	} \
232	} while (0)
233
234	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
235
236
237	/*********************************************************************************************************************************
238	* Structures and Typedefs *
239	*********************************************************************************************************************************/
240	/**
241	* VMX transient state.
242	*
243	* A state structure for holding miscellaneous information across
244	* VMX non-root operation and restored after the transition.
245	*/
246	typedef struct VMXTRANSIENT
247	{
248	/** The host's rflags/eflags. */
249	RTCCUINTREG fEFlags;
250	#if HC_ARCH_BITS == 32
251	uint32_t u32Alignment0;
252	#endif
253	/** The guest's TPR value used for TPR shadowing. */
254	uint8_t u8GuestTpr;
255	/** Alignment. */
256	uint8_t abAlignment0[7];
257
258	/** The basic VM-exit reason. */
259	uint16_t uExitReason;
260	/** Alignment. */
261	uint16_t u16Alignment0;
262	/** The VM-exit interruption error code. */
263	uint32_t uExitIntErrorCode;
264	/** The VM-exit exit code qualification. */
265	uint64_t uExitQual;
266	/** The Guest-linear address. */
267	uint64_t uGuestLinearAddr;
268
269	/** The VM-exit interruption-information field. */
270	uint32_t uExitIntInfo;
271	/** The VM-exit instruction-length field. */
272	uint32_t cbInstr;
273	/** The VM-exit instruction-information field. */
274	VMXEXITINSTRINFO ExitInstrInfo;
275	/** Whether the VM-entry failed or not. */
276	bool fVMEntryFailed;
277	/** Alignment. */
278	uint8_t abAlignment1[3];
279
280	/** The VM-entry interruption-information field. */
281	uint32_t uEntryIntInfo;
282	/** The VM-entry exception error code field. */
283	uint32_t uEntryXcptErrorCode;
284	/** The VM-entry instruction length field. */
285	uint32_t cbEntryInstr;
286
287	/** IDT-vectoring information field. */
288	uint32_t uIdtVectoringInfo;
289	/** IDT-vectoring error code. */
290	uint32_t uIdtVectoringErrorCode;
291
292	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
293	uint32_t fVmcsFieldsRead;
294
295	/** Whether the guest debug state was active at the time of VM-exit. */
296	bool fWasGuestDebugStateActive;
297	/** Whether the hyper debug state was active at the time of VM-exit. */
298	bool fWasHyperDebugStateActive;
299	/** Whether TSC-offsetting should be setup before VM-entry. */
300	bool fUpdateTscOffsettingAndPreemptTimer;
301	/** Whether the VM-exit was caused by a page-fault during delivery of a
302	* contributory exception or a page-fault. */
303	bool fVectoringDoublePF;
304	/** Whether the VM-exit was caused by a page-fault during delivery of an
305	* external interrupt or NMI. */
306	bool fVectoringPF;
307	} VMXTRANSIENT;
308	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
309	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
310	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
311	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
312	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
313	/** Pointer to VMX transient state. */
314	typedef VMXTRANSIENT *PVMXTRANSIENT;
315
316	/**
317	* Memory operand read or write access.
318	*/
319	typedef enum VMXMEMACCESS
320	{
321	VMXMEMACCESS_READ = 0,
322	VMXMEMACCESS_WRITE = 1
323	} VMXMEMACCESS;
324
325	/**
326	* VMX VM-exit handler.
327	*
328	* @returns Strict VBox status code (i.e. informational status codes too).
329	* @param pVCpu The cross context virtual CPU structure.
330	* @param pVmxTransient Pointer to the VMX-transient structure.
331	*/
332	#ifndef HMVMX_USE_FUNCTION_TABLE
333	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
334	#else
335	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
336	/** Pointer to VM-exit handler. */
337	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
338	#endif
339
340	/**
341	* VMX VM-exit handler, non-strict status code.
342	*
343	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
344	*
345	* @returns VBox status code, no informational status code returned.
346	* @param pVCpu The cross context virtual CPU structure.
347	* @param pVmxTransient Pointer to the VMX-transient structure.
348	*
349	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
350	* use of that status code will be replaced with VINF_EM_SOMETHING
351	* later when switching over to IEM.
352	*/
353	#ifndef HMVMX_USE_FUNCTION_TABLE
354	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
355	#else
356	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
357	#endif
358
359
360	/*********************************************************************************************************************************
361	* Internal Functions *
362	*********************************************************************************************************************************/
363	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush);
364	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr);
365	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
366	static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
367	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
368	RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
369	#if HC_ARCH_BITS == 32
370	static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
371	#endif
372	#ifndef HMVMX_USE_FUNCTION_TABLE
373	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
374	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
375	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
376	#else
377	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
378	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
379	#endif
380
381	/** @name VM-exit handlers.
382	* @{
383	*/
384	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
385	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
386	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
387	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
388	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
389	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
390	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
391	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
392	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
393	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
394	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
395	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
396	static FNVMXEXITHANDLER hmR0VmxExitHlt;
397	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
398	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
399	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
400	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
401	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
402	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
403	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
404	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
405	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
406	static FNVMXEXITHANDLER hmR0VmxExitVmread;
407	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
408	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
409	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
410	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
411	#endif
412	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
413	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
414	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
415	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
416	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
417	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
418	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
419	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
420	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
421	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
422	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
423	static FNVMXEXITHANDLER hmR0VmxExitMwait;
424	static FNVMXEXITHANDLER hmR0VmxExitMtf;
425	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
426	static FNVMXEXITHANDLER hmR0VmxExitPause;
427	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
428	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
429	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
430	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
431	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
432	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
433	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
434	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
435	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
436	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
437	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
438	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
439	/** @} */
440
441	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
442	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
443	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
444	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
445	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
446	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
447	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
448	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu);
449
450
451	/*********************************************************************************************************************************
452	* Global Variables *
453	*********************************************************************************************************************************/
454	#ifdef HMVMX_USE_FUNCTION_TABLE
455
456	/**
457	* VMX_EXIT dispatch table.
458	*/
459	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
460	{
461	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
462	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
463	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
464	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
465	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
466	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
467	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
468	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
469	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
470	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
471	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
472	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
473	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
474	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
475	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
476	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
477	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
478	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
479	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
480	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
481	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
482	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
483	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
484	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
485	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
486	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
487	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
488	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
489	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
490	#else
491	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
492	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
493	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
494	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
495	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
496	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
497	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
498	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
499	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
500	#endif
501	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
502	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
503	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
504	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
505	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
506	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
507	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
508	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
509	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
510	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
511	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
512	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
513	/* 40 UNDEFINED */ hmR0VmxExitPause,
514	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
515	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
516	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
517	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
518	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
519	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitXdtrAccess,
520	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitXdtrAccess,
521	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
522	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
523	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
524	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
525	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
526	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
527	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
528	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
529	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
530	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
531	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
532	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
533	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
534	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
535	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
536	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
537	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
538	};
539	#endif /* HMVMX_USE_FUNCTION_TABLE */
540
541	#ifdef VBOX_STRICT
542	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
543	{
544	/* 0 */ "(Not Used)",
545	/* 1 */ "VMCALL executed in VMX root operation.",
546	/* 2 */ "VMCLEAR with invalid physical address.",
547	/* 3 */ "VMCLEAR with VMXON pointer.",
548	/* 4 */ "VMLAUNCH with non-clear VMCS.",
549	/* 5 */ "VMRESUME with non-launched VMCS.",
550	/* 6 */ "VMRESUME after VMXOFF",
551	/* 7 */ "VM-entry with invalid control fields.",
552	/* 8 */ "VM-entry with invalid host state fields.",
553	/* 9 */ "VMPTRLD with invalid physical address.",
554	/* 10 */ "VMPTRLD with VMXON pointer.",
555	/* 11 */ "VMPTRLD with incorrect revision identifier.",
556	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
557	/* 13 */ "VMWRITE to read-only VMCS component.",
558	/* 14 */ "(Not Used)",
559	/* 15 */ "VMXON executed in VMX root operation.",
560	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
561	/* 17 */ "VM-entry with non-launched executing VMCS.",
562	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
563	/* 19 */ "VMCALL with non-clear VMCS.",
564	/* 20 */ "VMCALL with invalid VM-exit control fields.",
565	/* 21 */ "(Not Used)",
566	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
567	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
568	/* 24 */ "VMCALL with invalid SMM-monitor features.",
569	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
570	/* 26 */ "VM-entry with events blocked by MOV SS.",
571	/* 27 */ "(Not Used)",
572	/* 28 */ "Invalid operand to INVEPT/INVVPID."
573	};
574	#endif /* VBOX_STRICT */
575
576
577	/**
578	* Updates the VM's last error record.
579	*
580	* If there was a VMX instruction error, reads the error data from the VMCS and
581	* updates VCPU's last error record as well.
582	*
583	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
584	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
585	* VERR_VMX_INVALID_VMCS_FIELD.
586	* @param rc The error code.
587	*/
588	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
589	{
590	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
591	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
592	{
593	AssertPtrReturnVoid(pVCpu);
594	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
595	}
596	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
597	}
598
599
600	/**
601	* Reads the VM-entry interruption-information field from the VMCS into the VMX
602	* transient structure.
603	*
604	* @returns VBox status code.
605	* @param pVmxTransient Pointer to the VMX transient structure.
606	*
607	* @remarks No-long-jump zone!!!
608	*/
609	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
610	{
611	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
612	AssertRCReturn(rc, rc);
613	return VINF_SUCCESS;
614	}
615
616	#ifdef VBOX_STRICT
617	/**
618	* Reads the VM-entry exception error code field from the VMCS into
619	* the VMX transient structure.
620	*
621	* @returns VBox status code.
622	* @param pVmxTransient Pointer to the VMX transient structure.
623	*
624	* @remarks No-long-jump zone!!!
625	*/
626	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
627	{
628	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
629	AssertRCReturn(rc, rc);
630	return VINF_SUCCESS;
631	}
632
633
634	/**
635	* Reads the VM-entry exception error code field from the VMCS into
636	* the VMX transient structure.
637	*
638	* @returns VBox status code.
639	* @param pVmxTransient Pointer to the VMX transient structure.
640	*
641	* @remarks No-long-jump zone!!!
642	*/
643	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
644	{
645	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
646	AssertRCReturn(rc, rc);
647	return VINF_SUCCESS;
648	}
649	#endif /* VBOX_STRICT */
650
651
652	/**
653	* Reads the VM-exit interruption-information field from the VMCS into the VMX
654	* transient structure.
655	*
656	* @returns VBox status code.
657	* @param pVmxTransient Pointer to the VMX transient structure.
658	*/
659	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
660	{
661	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
662	{
663	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
664	AssertRCReturn(rc,rc);
665	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
666	}
667	return VINF_SUCCESS;
668	}
669
670
671	/**
672	* Reads the VM-exit interruption error code from the VMCS into the VMX
673	* transient structure.
674	*
675	* @returns VBox status code.
676	* @param pVmxTransient Pointer to the VMX transient structure.
677	*/
678	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
679	{
680	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
681	{
682	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
683	AssertRCReturn(rc, rc);
684	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
685	}
686	return VINF_SUCCESS;
687	}
688
689
690	/**
691	* Reads the VM-exit instruction length field from the VMCS into the VMX
692	* transient structure.
693	*
694	* @returns VBox status code.
695	* @param pVmxTransient Pointer to the VMX transient structure.
696	*/
697	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
698	{
699	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
700	{
701	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
702	AssertRCReturn(rc, rc);
703	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
704	}
705	return VINF_SUCCESS;
706	}
707
708
709	/**
710	* Reads the VM-exit instruction-information field from the VMCS into
711	* the VMX transient structure.
712	*
713	* @returns VBox status code.
714	* @param pVmxTransient Pointer to the VMX transient structure.
715	*/
716	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
717	{
718	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
719	{
720	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
721	AssertRCReturn(rc, rc);
722	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
723	}
724	return VINF_SUCCESS;
725	}
726
727
728	/**
729	* Reads the VM-exit Qualification from the VMCS into the VMX transient structure.
730	*
731	* @returns VBox status code.
732	* @param pVCpu The cross context virtual CPU structure of the
733	* calling EMT. (Required for the VMCS cache case.)
734	* @param pVmxTransient Pointer to the VMX transient structure.
735	*/
736	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
737	{
738	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
739	{
740	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
741	AssertRCReturn(rc, rc);
742	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
743	}
744	return VINF_SUCCESS;
745	}
746
747
748	/**
749	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
750	*
751	* @returns VBox status code.
752	* @param pVCpu The cross context virtual CPU structure of the
753	* calling EMT. (Required for the VMCS cache case.)
754	* @param pVmxTransient Pointer to the VMX transient structure.
755	*/
756	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
757	{
758	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
759	{
760	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
761	AssertRCReturn(rc, rc);
762	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
763	}
764	return VINF_SUCCESS;
765	}
766
767
768	/**
769	* Reads the IDT-vectoring information field from the VMCS into the VMX
770	* transient structure.
771	*
772	* @returns VBox status code.
773	* @param pVmxTransient Pointer to the VMX transient structure.
774	*
775	* @remarks No-long-jump zone!!!
776	*/
777	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
778	{
779	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
780	{
781	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
782	AssertRCReturn(rc, rc);
783	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
784	}
785	return VINF_SUCCESS;
786	}
787
788
789	/**
790	* Reads the IDT-vectoring error code from the VMCS into the VMX
791	* transient structure.
792	*
793	* @returns VBox status code.
794	* @param pVmxTransient Pointer to the VMX transient structure.
795	*/
796	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
797	{
798	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
799	{
800	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
801	AssertRCReturn(rc, rc);
802	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
803	}
804	return VINF_SUCCESS;
805	}
806
807
808	/**
809	* Enters VMX root mode operation on the current CPU.
810	*
811	* @returns VBox status code.
812	* @param pVM The cross context VM structure. Can be
813	* NULL, after a resume.
814	* @param HCPhysCpuPage Physical address of the VMXON region.
815	* @param pvCpuPage Pointer to the VMXON region.
816	*/
817	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
818	{
819	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
820	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
821	Assert(pvCpuPage);
822	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
823
824	if (pVM)
825	{
826	/* Write the VMCS revision dword to the VMXON region. */
827	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
828	}
829
830	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
831	RTCCUINTREG fEFlags = ASMIntDisableFlags();
832
833	/* Enable the VMX bit in CR4 if necessary. */
834	RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
835
836	/* Enter VMX root mode. */
837	int rc = VMXEnable(HCPhysCpuPage);
838	if (RT_FAILURE(rc))
839	{
840	if (!(uOldCr4 & X86_CR4_VMXE))
841	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
842
843	if (pVM)
844	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
845	}
846
847	/* Restore interrupts. */
848	ASMSetFlags(fEFlags);
849	return rc;
850	}
851
852
853	/**
854	* Exits VMX root mode operation on the current CPU.
855	*
856	* @returns VBox status code.
857	*/
858	static int hmR0VmxLeaveRootMode(void)
859	{
860	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
861
862	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
863	RTCCUINTREG fEFlags = ASMIntDisableFlags();
864
865	/* If we're for some reason not in VMX root mode, then don't leave it. */
866	RTCCUINTREG uHostCR4 = ASMGetCR4();
867
868	int rc;
869	if (uHostCR4 & X86_CR4_VMXE)
870	{
871	/* Exit VMX root mode and clear the VMX bit in CR4. */
872	VMXDisable();
873	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
874	rc = VINF_SUCCESS;
875	}
876	else
877	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
878
879	/* Restore interrupts. */
880	ASMSetFlags(fEFlags);
881	return rc;
882	}
883
884
885	/**
886	* Allocates and maps one physically contiguous page. The allocated page is
887	* zero'd out. (Used by various VT-x structures).
888	*
889	* @returns IPRT status code.
890	* @param pMemObj Pointer to the ring-0 memory object.
891	* @param ppVirt Where to store the virtual address of the
892	* allocation.
893	* @param pHCPhys Where to store the physical address of the
894	* allocation.
895	*/
896	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
897	{
898	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
899	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
900	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
901
902	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
903	if (RT_FAILURE(rc))
904	return rc;
905	ppVirt = RTR0MemObjAddress(pMemObj);
906	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
907	ASMMemZero32(*ppVirt, PAGE_SIZE);
908	return VINF_SUCCESS;
909	}
910
911
912	/**
913	* Frees and unmaps an allocated physical page.
914	*
915	* @param pMemObj Pointer to the ring-0 memory object.
916	* @param ppVirt Where to re-initialize the virtual address of
917	* allocation as 0.
918	* @param pHCPhys Where to re-initialize the physical address of the
919	* allocation as 0.
920	*/
921	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
922	{
923	AssertPtr(pMemObj);
924	AssertPtr(ppVirt);
925	AssertPtr(pHCPhys);
926	if (*pMemObj != NIL_RTR0MEMOBJ)
927	{
928	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
929	AssertRC(rc);
930	*pMemObj = NIL_RTR0MEMOBJ;
931	*ppVirt = 0;
932	*pHCPhys = 0;
933	}
934	}
935
936
937	/**
938	* Worker function to free VT-x related structures.
939	*
940	* @returns IPRT status code.
941	* @param pVM The cross context VM structure.
942	*/
943	static void hmR0VmxStructsFree(PVM pVM)
944	{
945	for (VMCPUID i = 0; i < pVM->cCpus; i++)
946	{
947	PVMCPU pVCpu = &pVM->aCpus[i];
948	AssertPtr(pVCpu);
949
950	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
951	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
952
953	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
954	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
955
956	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
957	}
958
959	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
960	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
961	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
962	#endif
963	}
964
965
966	/**
967	* Worker function to allocate VT-x related VM structures.
968	*
969	* @returns IPRT status code.
970	* @param pVM The cross context VM structure.
971	*/
972	static int hmR0VmxStructsAlloc(PVM pVM)
973	{
974	/*
975	* Initialize members up-front so we can cleanup properly on allocation failure.
976	*/
977	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
978	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
979	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
980	pVM->hm.s.vmx.HCPhys##a_Name = 0;
981
982	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
983	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
984	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
985	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
986
987	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
988	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
989	#endif
990	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
991
992	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
993	for (VMCPUID i = 0; i < pVM->cCpus; i++)
994	{
995	PVMCPU pVCpu = &pVM->aCpus[i];
996	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
997	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
998	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
999	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
1000	}
1001	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
1002	#undef VMXLOCAL_INIT_VM_MEMOBJ
1003
1004	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
1005	AssertReturnStmt(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE) <= PAGE_SIZE,
1006	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
1007	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
1008
1009	/*
1010	* Allocate all the VT-x structures.
1011	*/
1012	int rc = VINF_SUCCESS;
1013	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1014	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1015	if (RT_FAILURE(rc))
1016	goto cleanup;
1017	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1018	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1019	#endif
1020
1021	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1022	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1023	{
1024	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1025	&pVM->hm.s.vmx.HCPhysApicAccess);
1026	if (RT_FAILURE(rc))
1027	goto cleanup;
1028	}
1029
1030	/*
1031	* Initialize per-VCPU VT-x structures.
1032	*/
1033	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1034	{
1035	PVMCPU pVCpu = &pVM->aCpus[i];
1036	AssertPtr(pVCpu);
1037
1038	/* Allocate the VM control structure (VMCS). */
1039	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1040	if (RT_FAILURE(rc))
1041	goto cleanup;
1042
1043	/* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1044	if ( PDMHasApic(pVM)
1045	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1046	{
1047	rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1048	NULL /* pR3Ptr /, NULL / pRCPtr */);
1049	if (RT_FAILURE(rc))
1050	goto cleanup;
1051	}
1052
1053	/*
1054	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1055	* transparent accesses of specific MSRs.
1056	*
1057	* If the condition for enabling MSR bitmaps changes here, don't forget to
1058	* update HMAreMsrBitmapsAvailable().
1059	*/
1060	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1061	{
1062	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1063	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1064	if (RT_FAILURE(rc))
1065	goto cleanup;
1066	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1067	}
1068
1069	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1070	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1071	if (RT_FAILURE(rc))
1072	goto cleanup;
1073
1074	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1075	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1076	if (RT_FAILURE(rc))
1077	goto cleanup;
1078	}
1079
1080	return VINF_SUCCESS;
1081
1082	cleanup:
1083	hmR0VmxStructsFree(pVM);
1084	return rc;
1085	}
1086
1087
1088	/**
1089	* Does global VT-x initialization (called during module initialization).
1090	*
1091	* @returns VBox status code.
1092	*/
1093	VMMR0DECL(int) VMXR0GlobalInit(void)
1094	{
1095	#ifdef HMVMX_USE_FUNCTION_TABLE
1096	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1097	# ifdef VBOX_STRICT
1098	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1099	Assert(g_apfnVMExitHandlers[i]);
1100	# endif
1101	#endif
1102	return VINF_SUCCESS;
1103	}
1104
1105
1106	/**
1107	* Does global VT-x termination (called during module termination).
1108	*/
1109	VMMR0DECL(void) VMXR0GlobalTerm()
1110	{
1111	/* Nothing to do currently. */
1112	}
1113
1114
1115	/**
1116	* Sets up and activates VT-x on the current CPU.
1117	*
1118	* @returns VBox status code.
1119	* @param pHostCpu Pointer to the global CPU info struct.
1120	* @param pVM The cross context VM structure. Can be
1121	* NULL after a host resume operation.
1122	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1123	* fEnabledByHost is @c true).
1124	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1125	* @a fEnabledByHost is @c true).
1126	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1127	* enable VT-x on the host.
1128	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1129	*/
1130	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1131	void *pvMsrs)
1132	{
1133	Assert(pHostCpu);
1134	Assert(pvMsrs);
1135	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1136
1137	/* Enable VT-x if it's not already enabled by the host. */
1138	if (!fEnabledByHost)
1139	{
1140	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1141	if (RT_FAILURE(rc))
1142	return rc;
1143	}
1144
1145	/*
1146	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1147	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1148	* invalidated when flushing by VPID.
1149	*/
1150	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1151	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1152	{
1153	hmR0VmxFlushEpt(NULL /* pVCpu */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
1154	pHostCpu->fFlushAsidBeforeUse = false;
1155	}
1156	else
1157	pHostCpu->fFlushAsidBeforeUse = true;
1158
1159	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1160	++pHostCpu->cTlbFlushes;
1161
1162	return VINF_SUCCESS;
1163	}
1164
1165
1166	/**
1167	* Deactivates VT-x on the current CPU.
1168	*
1169	* @returns VBox status code.
1170	* @param pHostCpu Pointer to the global CPU info struct.
1171	* @param pvCpuPage Pointer to the VMXON region.
1172	* @param HCPhysCpuPage Physical address of the VMXON region.
1173	*
1174	* @remarks This function should never be called when SUPR0EnableVTx() or
1175	* similar was used to enable VT-x on the host.
1176	*/
1177	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1178	{
1179	RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1180
1181	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1182	return hmR0VmxLeaveRootMode();
1183	}
1184
1185
1186	/**
1187	* Sets the permission bits for the specified MSR in the MSR bitmap.
1188	*
1189	* @param pVCpu The cross context virtual CPU structure.
1190	* @param uMsr The MSR value.
1191	* @param enmRead Whether reading this MSR causes a VM-exit.
1192	* @param enmWrite Whether writing this MSR causes a VM-exit.
1193	*/
1194	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1195	{
1196	int32_t iBit;
1197	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1198
1199	/*
1200	* MSR Layout:
1201	* Byte index MSR range Interpreted as
1202	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1203	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1204	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1205	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1206	*
1207	* A bit corresponding to an MSR within the above range causes a VM-exit
1208	* if the bit is 1 on executions of RDMSR/WRMSR.
1209	*
1210	* If an MSR falls out of the MSR range, it always cause a VM-exit.
1211	*
1212	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
1213	*/
1214	if (uMsr <= 0x00001fff)
1215	iBit = uMsr;
1216	else if (uMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1217	{
1218	iBit = uMsr - UINT32_C(0xc0000000);
1219	pbMsrBitmap += 0x400;
1220	}
1221	else
1222	AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1223
1224	Assert(iBit <= 0x1fff);
1225	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1226	ASMBitSet(pbMsrBitmap, iBit);
1227	else
1228	ASMBitClear(pbMsrBitmap, iBit);
1229
1230	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1231	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1232	else
1233	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1234	}
1235
1236
1237	/**
1238	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1239	* area.
1240	*
1241	* @returns VBox status code.
1242	* @param pVCpu The cross context virtual CPU structure.
1243	* @param cMsrs The number of MSRs.
1244	*/
1245	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1246	{
1247	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1248	uint64_t const uVmxMiscMsr = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc;
1249	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(uVmxMiscMsr);
1250	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1251	{
1252	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1253	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1254	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1255	}
1256
1257	/* Update number of guest MSRs to load/store across the world-switch. */
1258	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1259	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1260
1261	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1262	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1263	AssertRCReturn(rc, rc);
1264
1265	/* Update the VCPU's copy of the MSR count. */
1266	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1267
1268	return VINF_SUCCESS;
1269	}
1270
1271
1272	/**
1273	* Adds a new (or updates the value of an existing) guest/host MSR
1274	* pair to be swapped during the world-switch as part of the
1275	* auto-load/store MSR area in the VMCS.
1276	*
1277	* @returns VBox status code.
1278	* @param pVCpu The cross context virtual CPU structure.
1279	* @param uMsr The MSR.
1280	* @param uGuestMsrValue Value of the guest MSR.
1281	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1282	* necessary.
1283	* @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1284	* its value was updated. Optional, can be NULL.
1285	*/
1286	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1287	bool *pfAddedAndUpdated)
1288	{
1289	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1290	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1291	uint32_t i;
1292	for (i = 0; i < cMsrs; i++)
1293	{
1294	if (pGuestMsr->u32Msr == uMsr)
1295	break;
1296	pGuestMsr++;
1297	}
1298
1299	bool fAdded = false;
1300	if (i == cMsrs)
1301	{
1302	++cMsrs;
1303	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1304	AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1305
1306	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1307	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1308	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1309
1310	fAdded = true;
1311	}
1312
1313	/* Update the MSR values in the auto-load/store MSR area. */
1314	pGuestMsr->u32Msr = uMsr;
1315	pGuestMsr->u64Value = uGuestMsrValue;
1316
1317	/* Create/update the MSR slot in the host MSR area. */
1318	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1319	pHostMsr += i;
1320	pHostMsr->u32Msr = uMsr;
1321
1322	/*
1323	* Update the host MSR only when requested by the caller AND when we're
1324	* adding it to the auto-load/store area. Otherwise, it would have been
1325	* updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1326	*/
1327	bool fUpdatedMsrValue = false;
1328	if ( fAdded
1329	&& fUpdateHostMsr)
1330	{
1331	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1332	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1333	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1334	fUpdatedMsrValue = true;
1335	}
1336
1337	if (pfAddedAndUpdated)
1338	*pfAddedAndUpdated = fUpdatedMsrValue;
1339	return VINF_SUCCESS;
1340	}
1341
1342
1343	/**
1344	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1345	* auto-load/store MSR area in the VMCS.
1346	*
1347	* @returns VBox status code.
1348	* @param pVCpu The cross context virtual CPU structure.
1349	* @param uMsr The MSR.
1350	*/
1351	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1352	{
1353	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1354	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1355	for (uint32_t i = 0; i < cMsrs; i++)
1356	{
1357	/* Find the MSR. */
1358	if (pGuestMsr->u32Msr == uMsr)
1359	{
1360	/* If it's the last MSR, simply reduce the count. */
1361	if (i == cMsrs - 1)
1362	{
1363	--cMsrs;
1364	break;
1365	}
1366
1367	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1368	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1369	pLastGuestMsr += cMsrs - 1;
1370	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1371	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1372
1373	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1374	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1375	pLastHostMsr += cMsrs - 1;
1376	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1377	pHostMsr->u64Value = pLastHostMsr->u64Value;
1378	--cMsrs;
1379	break;
1380	}
1381	pGuestMsr++;
1382	}
1383
1384	/* Update the VMCS if the count changed (meaning the MSR was found). */
1385	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1386	{
1387	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1388	AssertRCReturn(rc, rc);
1389
1390	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1391	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1392	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1393
1394	Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1395	return VINF_SUCCESS;
1396	}
1397
1398	return VERR_NOT_FOUND;
1399	}
1400
1401
1402	/**
1403	* Checks if the specified guest MSR is part of the auto-load/store area in
1404	* the VMCS.
1405	*
1406	* @returns true if found, false otherwise.
1407	* @param pVCpu The cross context virtual CPU structure.
1408	* @param uMsr The MSR to find.
1409	*/
1410	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1411	{
1412	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1413	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1414
1415	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1416	{
1417	if (pGuestMsr->u32Msr == uMsr)
1418	return true;
1419	}
1420	return false;
1421	}
1422
1423
1424	/**
1425	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1426	*
1427	* @param pVCpu The cross context virtual CPU structure.
1428	*
1429	* @remarks No-long-jump zone!!!
1430	*/
1431	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1432	{
1433	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1434	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1435	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1436	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1437
1438	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1439	{
1440	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1441
1442	/*
1443	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1444	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1445	*/
1446	if (pHostMsr->u32Msr == MSR_K6_EFER)
1447	pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1448	else
1449	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1450	}
1451
1452	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1453	}
1454
1455
1456	/**
1457	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1458	* perform lazy restoration of the host MSRs while leaving VT-x.
1459	*
1460	* @param pVCpu The cross context virtual CPU structure.
1461	*
1462	* @remarks No-long-jump zone!!!
1463	*/
1464	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1465	{
1466	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1467
1468	/*
1469	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1470	*/
1471	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1472	{
1473	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1474	#if HC_ARCH_BITS == 64
1475	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1476	{
1477	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1478	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1479	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1480	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1481	}
1482	#endif
1483	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1484	}
1485	}
1486
1487
1488	/**
1489	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1490	* lazily while leaving VT-x.
1491	*
1492	* @returns true if it does, false otherwise.
1493	* @param pVCpu The cross context virtual CPU structure.
1494	* @param uMsr The MSR to check.
1495	*/
1496	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1497	{
1498	NOREF(pVCpu);
1499	#if HC_ARCH_BITS == 64
1500	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1501	{
1502	switch (uMsr)
1503	{
1504	case MSR_K8_LSTAR:
1505	case MSR_K6_STAR:
1506	case MSR_K8_SF_MASK:
1507	case MSR_K8_KERNEL_GS_BASE:
1508	return true;
1509	}
1510	}
1511	#else
1512	RT_NOREF(pVCpu, uMsr);
1513	#endif
1514	return false;
1515	}
1516
1517
1518	/**
1519	* Loads a set of guests MSRs to allow read/passthru to the guest.
1520	*
1521	* The name of this function is slightly confusing. This function does NOT
1522	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1523	* common prefix for functions dealing with "lazy restoration" of the shared
1524	* MSRs.
1525	*
1526	* @param pVCpu The cross context virtual CPU structure.
1527	*
1528	* @remarks No-long-jump zone!!!
1529	*/
1530	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1531	{
1532	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1533	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1534
1535	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1536	#if HC_ARCH_BITS == 64
1537	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1538	{
1539	/*
1540	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1541	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1542	* we can skip a few MSR writes.
1543	*
1544	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1545	* guest MSR values in the guest-CPU context might be different to what's currently
1546	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1547	* CPU, see @bugref{8728}.
1548	*/
1549	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1550	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1551	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1552	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1553	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1554	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1555	{
1556	#ifdef VBOX_STRICT
1557	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1558	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1559	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1560	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1561	#endif
1562	}
1563	else
1564	{
1565	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1566	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
1567	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
1568	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
1569	}
1570	}
1571	#endif
1572	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
1573	}
1574
1575
1576	/**
1577	* Performs lazy restoration of the set of host MSRs if they were previously
1578	* loaded with guest MSR values.
1579	*
1580	* @param pVCpu The cross context virtual CPU structure.
1581	*
1582	* @remarks No-long-jump zone!!!
1583	* @remarks The guest MSRs should have been saved back into the guest-CPU
1584	* context by hmR0VmxImportGuestState()!!!
1585	*/
1586	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1587	{
1588	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1589	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1590
1591	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1592	{
1593	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1594	#if HC_ARCH_BITS == 64
1595	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1596	{
1597	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1598	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1599	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1600	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1601	}
1602	#endif
1603	}
1604	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
1605	}
1606
1607
1608	/**
1609	* Verifies that our cached values of the VMCS fields are all consistent with
1610	* what's actually present in the VMCS.
1611	*
1612	* @returns VBox status code.
1613	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
1614	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
1615	* VMCS content. HMCPU error-field is
1616	* updated, see VMX_VCI_XXX.
1617	* @param pVCpu The cross context virtual CPU structure.
1618	*/
1619	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1620	{
1621	uint32_t u32Val;
1622	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1623	AssertRCReturn(rc, rc);
1624	AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32EntryCtls == u32Val,
1625	("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1626	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
1627	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1628
1629	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1630	AssertRCReturn(rc, rc);
1631	AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ExitCtls == u32Val,
1632	("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1633	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
1634	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1635
1636	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1637	AssertRCReturn(rc, rc);
1638	AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32PinCtls == u32Val,
1639	("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1640	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
1641	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1642
1643	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1644	AssertRCReturn(rc, rc);
1645	AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls == u32Val,
1646	("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1647	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
1648	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1649
1650	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
1651	{
1652	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1653	AssertRCReturn(rc, rc);
1654	AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1655	("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1656	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
1657	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1658	}
1659
1660	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
1661	AssertRCReturn(rc, rc);
1662	AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32XcptBitmap == u32Val,
1663	("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap, u32Val),
1664	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
1665	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1666
1667	uint64_t u64Val;
1668	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
1669	AssertRCReturn(rc, rc);
1670	AssertMsgReturnStmt(pVCpu->hm.s.vmx.u64TscOffset == u64Val,
1671	("Cache=%#RX64 VMCS=%#RX64\n", pVCpu->hm.s.vmx.u64TscOffset, u64Val),
1672	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
1673	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1674
1675	return VINF_SUCCESS;
1676	}
1677
1678
1679	#ifdef VBOX_STRICT
1680	/**
1681	* Verifies that our cached host EFER value has not changed
1682	* since we cached it.
1683	*
1684	* @param pVCpu The cross context virtual CPU structure.
1685	*/
1686	static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1687	{
1688	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1689
1690	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
1691	{
1692	uint64_t u64Val;
1693	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1694	AssertRC(rc);
1695
1696	uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1697	AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1698	}
1699	}
1700
1701
1702	/**
1703	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1704	* VMCS are correct.
1705	*
1706	* @param pVCpu The cross context virtual CPU structure.
1707	*/
1708	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1709	{
1710	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1711
1712	/* Verify MSR counts in the VMCS are what we think it should be. */
1713	uint32_t cMsrs;
1714	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1715	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1716
1717	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1718	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1719
1720	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1721	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1722
1723	PCVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1724	PCVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1725	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1726	{
1727	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1728	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1729	pGuestMsr->u32Msr, cMsrs));
1730
1731	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1732	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1733	pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1734
1735	/* Verify that the permissions are as expected in the MSR bitmap. */
1736	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1737	{
1738	VMXMSREXITREAD enmRead;
1739	VMXMSREXITWRITE enmWrite;
1740	rc = HMVmxGetMsrPermission(pVCpu->hm.s.vmx.pvMsrBitmap, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1741	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("HMVmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1742	if (pGuestMsr->u32Msr == MSR_K6_EFER)
1743	{
1744	AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1745	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1746	}
1747	else
1748	{
1749	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1750	pGuestMsr->u32Msr, cMsrs));
1751	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1752	pGuestMsr->u32Msr, cMsrs));
1753	}
1754	}
1755	}
1756	}
1757	#endif /* VBOX_STRICT */
1758
1759
1760	/**
1761	* Flushes the TLB using EPT.
1762	*
1763	* @returns VBox status code.
1764	* @param pVCpu The cross context virtual CPU structure of the calling
1765	* EMT. Can be NULL depending on @a enmTlbFlush.
1766	* @param enmTlbFlush Type of flush.
1767	*
1768	* @remarks Caller is responsible for making sure this function is called only
1769	* when NestedPaging is supported and providing @a enmTlbFlush that is
1770	* supported by the CPU.
1771	* @remarks Can be called with interrupts disabled.
1772	*/
1773	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush)
1774	{
1775	uint64_t au64Descriptor[2];
1776	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
1777	au64Descriptor[0] = 0;
1778	else
1779	{
1780	Assert(pVCpu);
1781	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1782	}
1783	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1784
1785	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
1786	AssertMsg(rc == VINF_SUCCESS,
1787	("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0, rc));
1788
1789	if ( RT_SUCCESS(rc)
1790	&& pVCpu)
1791	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1792	}
1793
1794
1795	/**
1796	* Flushes the TLB using VPID.
1797	*
1798	* @returns VBox status code.
1799	* @param pVCpu The cross context virtual CPU structure of the calling
1800	* EMT. Can be NULL depending on @a enmTlbFlush.
1801	* @param enmTlbFlush Type of flush.
1802	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1803	* on @a enmTlbFlush).
1804	*
1805	* @remarks Can be called with interrupts disabled.
1806	*/
1807	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
1808	{
1809	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1810
1811	uint64_t au64Descriptor[2];
1812	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
1813	{
1814	au64Descriptor[0] = 0;
1815	au64Descriptor[1] = 0;
1816	}
1817	else
1818	{
1819	AssertPtr(pVCpu);
1820	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1821	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1822	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1823	au64Descriptor[1] = GCPtr;
1824	}
1825
1826	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
1827	AssertMsg(rc == VINF_SUCCESS,
1828	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1829
1830	if ( RT_SUCCESS(rc)
1831	&& pVCpu)
1832	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1833	NOREF(rc);
1834	}
1835
1836
1837	/**
1838	* Invalidates a guest page by guest virtual address. Only relevant for
1839	* EPT/VPID, otherwise there is nothing really to invalidate.
1840	*
1841	* @returns VBox status code.
1842	* @param pVCpu The cross context virtual CPU structure.
1843	* @param GCVirt Guest virtual address of the page to invalidate.
1844	*/
1845	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1846	{
1847	AssertPtr(pVCpu);
1848	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1849
1850	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1851	if (!fFlushPending)
1852	{
1853	/*
1854	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1855	* the EPT case. See @bugref{6043} and @bugref{6177}.
1856	*
1857	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1858	* as this function maybe called in a loop with individual addresses.
1859	*/
1860	PVM pVM = pVCpu->CTX_SUFF(pVM);
1861	if (pVM->hm.s.vmx.fVpid)
1862	{
1863	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1864
1865	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1866	/*
1867	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1868	* where executing INVVPID outside 64-bit mode does not flush translations of
1869	* 64-bit linear addresses, see @bugref{6208#c72}.
1870	*/
1871	if (RT_HI_U32(GCVirt))
1872	fVpidFlush = false;
1873	#endif
1874
1875	if (fVpidFlush)
1876	{
1877	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
1878	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1879	}
1880	else
1881	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1882	}
1883	else if (pVM->hm.s.fNestedPaging)
1884	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1885	}
1886
1887	return VINF_SUCCESS;
1888	}
1889
1890
1891	/**
1892	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1893	* case where neither EPT nor VPID is supported by the CPU.
1894	*
1895	* @param pVCpu The cross context virtual CPU structure.
1896	* @param pCpu Pointer to the global HM struct.
1897	*
1898	* @remarks Called with interrupts disabled.
1899	*/
1900	static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1901	{
1902	AssertPtr(pVCpu);
1903	AssertPtr(pCpu);
1904
1905	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1906
1907	Assert(pCpu->idCpu != NIL_RTCPUID);
1908	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1909	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1910	pVCpu->hm.s.fForceTLBFlush = false;
1911	return;
1912	}
1913
1914
1915	/**
1916	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1917	*
1918	* @param pVCpu The cross context virtual CPU structure.
1919	* @param pCpu Pointer to the global HM CPU struct.
1920	*
1921	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1922	* nomenclature. The reason is, to avoid confusion in compare statements
1923	* since the host-CPU copies are named "ASID".
1924	*
1925	* @remarks Called with interrupts disabled.
1926	*/
1927	static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1928	{
1929	#ifdef VBOX_WITH_STATISTICS
1930	bool fTlbFlushed = false;
1931	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1932	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1933	if (!fTlbFlushed) \
1934	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1935	} while (0)
1936	#else
1937	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1938	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1939	#endif
1940
1941	AssertPtr(pCpu);
1942	AssertPtr(pVCpu);
1943	Assert(pCpu->idCpu != NIL_RTCPUID);
1944
1945	PVM pVM = pVCpu->CTX_SUFF(pVM);
1946	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1947	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1948	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1949
1950	/*
1951	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
1952	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1953	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1954	* cannot reuse the current ASID anymore.
1955	*/
1956	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1957	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1958	{
1959	++pCpu->uCurrentAsid;
1960	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1961	{
1962	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1963	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1964	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1965	}
1966
1967	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1968	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1969	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1970
1971	/*
1972	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1973	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1974	*/
1975	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1976	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1977	HMVMX_SET_TAGGED_TLB_FLUSHED();
1978	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1979	}
1980	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
1981	{
1982	/*
1983	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1984	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1985	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1986	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1987	* mappings, see @bugref{6568}.
1988	*
1989	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1990	*/
1991	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1992	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1993	HMVMX_SET_TAGGED_TLB_FLUSHED();
1994	}
1995
1996	pVCpu->hm.s.fForceTLBFlush = false;
1997	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1998
1999	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2000	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2001	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2002	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2003	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2004	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2005	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2006	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2007	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2008
2009	/* Update VMCS with the VPID. */
2010	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2011	AssertRC(rc);
2012
2013	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2014	}
2015
2016
2017	/**
2018	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2019	*
2020	* @returns VBox status code.
2021	* @param pVCpu The cross context virtual CPU structure.
2022	* @param pCpu Pointer to the global HM CPU struct.
2023	*
2024	* @remarks Called with interrupts disabled.
2025	*/
2026	static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2027	{
2028	AssertPtr(pVCpu);
2029	AssertPtr(pCpu);
2030	Assert(pCpu->idCpu != NIL_RTCPUID);
2031	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2032	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2033
2034	/*
2035	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2036	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2037	*/
2038	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2039	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2040	{
2041	pVCpu->hm.s.fForceTLBFlush = true;
2042	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2043	}
2044
2045	/* Check for explicit TLB flushes. */
2046	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2047	{
2048	pVCpu->hm.s.fForceTLBFlush = true;
2049	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2050	}
2051
2052	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2053	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2054
2055	if (pVCpu->hm.s.fForceTLBFlush)
2056	{
2057	hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2058	pVCpu->hm.s.fForceTLBFlush = false;
2059	}
2060	}
2061
2062
2063	/**
2064	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2065	*
2066	* @returns VBox status code.
2067	* @param pVCpu The cross context virtual CPU structure.
2068	* @param pCpu Pointer to the global HM CPU struct.
2069	*
2070	* @remarks Called with interrupts disabled.
2071	*/
2072	static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2073	{
2074	AssertPtr(pVCpu);
2075	AssertPtr(pCpu);
2076	Assert(pCpu->idCpu != NIL_RTCPUID);
2077	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2078	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2079
2080	/*
2081	* Force a TLB flush for the first world switch if the current CPU differs from the one we
2082	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2083	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2084	* cannot reuse the current ASID anymore.
2085	*/
2086	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2087	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2088	{
2089	pVCpu->hm.s.fForceTLBFlush = true;
2090	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2091	}
2092
2093	/* Check for explicit TLB flushes. */
2094	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2095	{
2096	/*
2097	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2098	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2099	* fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2100	* include fExplicitFlush's too) - an obscure corner case.
2101	*/
2102	pVCpu->hm.s.fForceTLBFlush = true;
2103	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2104	}
2105
2106	PVM pVM = pVCpu->CTX_SUFF(pVM);
2107	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2108	if (pVCpu->hm.s.fForceTLBFlush)
2109	{
2110	++pCpu->uCurrentAsid;
2111	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2112	{
2113	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2114	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2115	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2116	}
2117
2118	pVCpu->hm.s.fForceTLBFlush = false;
2119	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2120	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2121	if (pCpu->fFlushAsidBeforeUse)
2122	{
2123	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2124	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2125	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2126	{
2127	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2128	pCpu->fFlushAsidBeforeUse = false;
2129	}
2130	else
2131	{
2132	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2133	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2134	}
2135	}
2136	}
2137
2138	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2139	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2140	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2141	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2142	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2143	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2144	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2145
2146	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2147	AssertRC(rc);
2148	}
2149
2150
2151	/**
2152	* Flushes the guest TLB entry based on CPU capabilities.
2153	*
2154	* @param pVCpu The cross context virtual CPU structure.
2155	* @param pCpu Pointer to the global HM CPU struct.
2156	*/
2157	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2158	{
2159	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2160	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2161	#endif
2162	PVM pVM = pVCpu->CTX_SUFF(pVM);
2163	switch (pVM->hm.s.vmx.enmTlbFlushType)
2164	{
2165	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2166	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2167	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2168	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2169	default:
2170	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2171	break;
2172	}
2173	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2174	}
2175
2176
2177	/**
2178	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2179	* TLB entries from the host TLB before VM-entry.
2180	*
2181	* @returns VBox status code.
2182	* @param pVM The cross context VM structure.
2183	*/
2184	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2185	{
2186	/*
2187	* Determine optimal flush type for Nested Paging.
2188	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2189	* guest execution (see hmR3InitFinalizeR0()).
2190	*/
2191	if (pVM->hm.s.fNestedPaging)
2192	{
2193	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2194	{
2195	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2196	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2197	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2198	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2199	else
2200	{
2201	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2202	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2203	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2204	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2205	}
2206
2207	/* Make sure the write-back cacheable memory type for EPT is supported. */
2208	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2209	{
2210	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2211	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2212	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2213	}
2214
2215	/* EPT requires a page-walk length of 4. */
2216	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2217	{
2218	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2219	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2220	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2221	}
2222	}
2223	else
2224	{
2225	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2226	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2227	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2228	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2229	}
2230	}
2231
2232	/*
2233	* Determine optimal flush type for VPID.
2234	*/
2235	if (pVM->hm.s.vmx.fVpid)
2236	{
2237	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2238	{
2239	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2240	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2241	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2242	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2243	else
2244	{
2245	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2246	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2247	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2248	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2249	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2250	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2251	pVM->hm.s.vmx.fVpid = false;
2252	}
2253	}
2254	else
2255	{
2256	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2257	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2258	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2259	pVM->hm.s.vmx.fVpid = false;
2260	}
2261	}
2262
2263	/*
2264	* Setup the handler for flushing tagged-TLBs.
2265	*/
2266	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2267	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2268	else if (pVM->hm.s.fNestedPaging)
2269	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2270	else if (pVM->hm.s.vmx.fVpid)
2271	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2272	else
2273	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2274	return VINF_SUCCESS;
2275	}
2276
2277
2278	/**
2279	* Sets up pin-based VM-execution controls in the VMCS.
2280	*
2281	* @returns VBox status code.
2282	* @param pVCpu The cross context virtual CPU structure.
2283	*
2284	* @remarks We don't really care about optimizing vmwrites here as it's done only
2285	* once per VM and hence we don't care about VMCS-field cache comparisons.
2286	*/
2287	static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2288	{
2289	PVM pVM = pVCpu->CTX_SUFF(pVM);
2290	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0; /* Bits set here must always be set. */
2291	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2292
2293	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2294	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2295
2296	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2297	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2298
2299	/* Enable the VMX preemption timer. */
2300	if (pVM->hm.s.vmx.fUsePreemptTimer)
2301	{
2302	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2303	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
2304	}
2305
2306	#if 0
2307	/* Enable posted-interrupt processing. */
2308	if (pVM->hm.s.fPostedIntrs)
2309	{
2310	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2311	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2312	fVal \|= VMX_PIN_CTL_POSTED_INT;
2313	}
2314	#endif
2315
2316	if ((fVal & fZap) != fVal)
2317	{
2318	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2319	pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0, fVal, fZap));
2320	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2321	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2322	}
2323
2324	/* Commit it to the VMCS and update our cache. */
2325	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2326	AssertRCReturn(rc, rc);
2327	pVCpu->hm.s.vmx.u32PinCtls = fVal;
2328
2329	return VINF_SUCCESS;
2330	}
2331
2332
2333	/**
2334	* Sets up secondary processor-based VM-execution controls in the VMCS.
2335	*
2336	* @returns VBox status code.
2337	* @param pVCpu The cross context virtual CPU structure.
2338	*
2339	* @remarks We don't really care about optimizing vmwrites here as it's done only
2340	* once per VM and hence we don't care about VMCS-field cache comparisons.
2341	*/
2342	static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2343	{
2344	PVM pVM = pVCpu->CTX_SUFF(pVM);
2345	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2346	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2347
2348	/* WBINVD causes a VM-exit. */
2349	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2350	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
2351
2352	/* Enable EPT (aka nested-paging). */
2353	if (pVM->hm.s.fNestedPaging)
2354	fVal \|= VMX_PROC_CTLS2_EPT;
2355
2356	/*
2357	* Enable the INVPCID instruction if supported by the hardware and we expose
2358	* it to the guest. Without this, guest executing INVPCID would cause a #UD.
2359	*/
2360	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2361	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
2362	fVal \|= VMX_PROC_CTLS2_INVPCID;
2363
2364	/* Enable VPID. */
2365	if (pVM->hm.s.vmx.fVpid)
2366	fVal \|= VMX_PROC_CTLS2_VPID;
2367
2368	/* Enable Unrestricted guest execution. */
2369	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2370	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
2371
2372	#if 0
2373	if (pVM->hm.s.fVirtApicRegs)
2374	{
2375	/* Enable APIC-register virtualization. */
2376	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
2377	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
2378
2379	/* Enable virtual-interrupt delivery. */
2380	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
2381	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
2382	}
2383	#endif
2384
2385	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
2386	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2387	* done dynamically. */
2388	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
2389	{
2390	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2391	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2392	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS; /* Virtualize APIC accesses. */
2393	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2394	AssertRCReturn(rc, rc);
2395	}
2396
2397	/* Enable RDTSCP. */
2398	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
2399	fVal \|= VMX_PROC_CTLS2_RDTSCP;
2400
2401	/* Enable Pause-Loop exiting. */
2402	if ( pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT
2403	&& pVM->hm.s.vmx.cPleGapTicks
2404	&& pVM->hm.s.vmx.cPleWindowTicks)
2405	{
2406	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
2407
2408	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2409	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2410	AssertRCReturn(rc, rc);
2411	}
2412
2413	if ((fVal & fZap) != fVal)
2414	{
2415	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2416	pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0, fVal, fZap));
2417	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2418	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2419	}
2420
2421	/* Commit it to the VMCS and update our cache. */
2422	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2423	AssertRCReturn(rc, rc);
2424	pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2425
2426	return VINF_SUCCESS;
2427	}
2428
2429
2430	/**
2431	* Sets up processor-based VM-execution controls in the VMCS.
2432	*
2433	* @returns VBox status code.
2434	* @param pVCpu The cross context virtual CPU structure.
2435	*
2436	* @remarks We don't really care about optimizing vmwrites here as it's done only
2437	* once per VM and hence we don't care about VMCS-field cache comparisons.
2438	*/
2439	static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2440	{
2441	PVM pVM = pVCpu->CTX_SUFF(pVM);
2442	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2443	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2444
2445	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
2446	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2447	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2448	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2449	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2450	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2451	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2452
2453	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2454	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
2455	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
2456	{
2457	LogRelFunc(("Unsupported VMX_PROC_CTLS_MOV_DR_EXIT combo!"));
2458	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2459	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2460	}
2461
2462	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2463	if (!pVM->hm.s.fNestedPaging)
2464	{
2465	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2466	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
2467	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
2468	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
2469	}
2470
2471	/* Use TPR shadowing if supported by the CPU. */
2472	if ( PDMHasApic(pVM)
2473	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
2474	{
2475	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2476	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2477	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2478	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2479	AssertRCReturn(rc, rc);
2480
2481	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2482	/* CR8 writes cause a VM-exit based on TPR threshold. */
2483	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
2484	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
2485	}
2486	else
2487	{
2488	/*
2489	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2490	* Set this control only for 64-bit guests.
2491	*/
2492	if (pVM->hm.s.fAllow64BitGuests)
2493	{
2494	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2495	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2496	}
2497	}
2498
2499	/* Use MSR-bitmaps if supported by the CPU. */
2500	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2501	{
2502	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
2503
2504	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2505	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2506	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2507	AssertRCReturn(rc, rc);
2508
2509	/*
2510	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2511	* automatically using dedicated fields in the VMCS.
2512	*/
2513	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2514	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2515	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2516	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2517	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2518	#if HC_ARCH_BITS == 64
2519	/*
2520	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2521	*/
2522	if (pVM->hm.s.fAllow64BitGuests)
2523	{
2524	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2525	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2526	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2527	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2528	}
2529	#endif
2530	/*
2531	* The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2532	* access (writes need to be executed without exiting, reds will #GP-fault anyway).
2533	*/
2534	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2535	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2536
2537	/* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2538	}
2539
2540	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2541	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2542	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
2543
2544	if ((fVal & fZap) != fVal)
2545	{
2546	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2547	pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0, fVal, fZap));
2548	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2549	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2550	}
2551
2552	/* Commit it to the VMCS and update our cache. */
2553	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2554	AssertRCReturn(rc, rc);
2555	pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2556
2557	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2558	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2559	return hmR0VmxSetupProcCtls2(pVCpu);
2560
2561	/* Sanity check, should not really happen. */
2562	if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2563	{
2564	LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2565	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2566	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2567	}
2568
2569	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2570	return VINF_SUCCESS;
2571	}
2572
2573
2574	/**
2575	* Sets up miscellaneous (everything other than Pin & Processor-based
2576	* VM-execution) control fields in the VMCS.
2577	*
2578	* @returns VBox status code.
2579	* @param pVCpu The cross context virtual CPU structure.
2580	*/
2581	static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2582	{
2583	AssertPtr(pVCpu);
2584
2585	int rc = VERR_GENERAL_FAILURE;
2586
2587	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2588	#if 0
2589	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2590	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2591	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2592
2593	/*
2594	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2595	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2596	* We thus use the exception bitmap to control it rather than use both.
2597	*/
2598	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2599	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2600
2601	/* All IO & IOIO instructions cause VM-exits. */
2602	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2603	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2604
2605	/* Initialize the MSR-bitmap area. */
2606	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2607	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2608	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2609	AssertRCReturn(rc, rc);
2610	#endif
2611
2612	/* Setup MSR auto-load/store area. */
2613	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2614	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2615	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2616	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2617	AssertRCReturn(rc, rc);
2618
2619	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2620	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2621	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2622	AssertRCReturn(rc, rc);
2623
2624	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2625	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2626	AssertRCReturn(rc, rc);
2627
2628	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2629	#if 0
2630	/* Setup debug controls */
2631	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2632	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2633	AssertRCReturn(rc, rc);
2634	#endif
2635
2636	return rc;
2637	}
2638
2639
2640	/**
2641	* Sets up the initial exception bitmap in the VMCS based on static conditions.
2642	*
2643	* We shall setup those exception intercepts that don't change during the
2644	* lifetime of the VM here. The rest are done dynamically while loading the
2645	* guest state.
2646	*
2647	* @returns VBox status code.
2648	* @param pVCpu The cross context virtual CPU structure.
2649	*/
2650	static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2651	{
2652	AssertPtr(pVCpu);
2653
2654	uint32_t uXcptBitmap;
2655
2656	/* Must always intercept #AC to prevent the guest from hanging the CPU. */
2657	uXcptBitmap = RT_BIT_32(X86_XCPT_AC);
2658
2659	/* Because we need to maintain the DR6 state even when intercepting DRx reads
2660	and writes, and because recursive #DBs can cause the CPU hang, we must always
2661	intercept #DB. */
2662	uXcptBitmap \|= RT_BIT_32(X86_XCPT_DB);
2663
2664	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2665	if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2666	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
2667
2668	/* Commit it to the VMCS. */
2669	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2670	AssertRCReturn(rc, rc);
2671
2672	/* Update our cache of the exception bitmap. */
2673	pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
2674	return VINF_SUCCESS;
2675	}
2676
2677
2678	/**
2679	* Does per-VM VT-x initialization.
2680	*
2681	* @returns VBox status code.
2682	* @param pVM The cross context VM structure.
2683	*/
2684	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2685	{
2686	LogFlowFunc(("pVM=%p\n", pVM));
2687
2688	int rc = hmR0VmxStructsAlloc(pVM);
2689	if (RT_FAILURE(rc))
2690	{
2691	LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2692	return rc;
2693	}
2694
2695	return VINF_SUCCESS;
2696	}
2697
2698
2699	/**
2700	* Does per-VM VT-x termination.
2701	*
2702	* @returns VBox status code.
2703	* @param pVM The cross context VM structure.
2704	*/
2705	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2706	{
2707	LogFlowFunc(("pVM=%p\n", pVM));
2708
2709	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2710	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2711	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2712	#endif
2713	hmR0VmxStructsFree(pVM);
2714	return VINF_SUCCESS;
2715	}
2716
2717
2718	/**
2719	* Sets up the VM for execution under VT-x.
2720	* This function is only called once per-VM during initialization.
2721	*
2722	* @returns VBox status code.
2723	* @param pVM The cross context VM structure.
2724	*/
2725	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2726	{
2727	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2728	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2729
2730	LogFlowFunc(("pVM=%p\n", pVM));
2731
2732	/*
2733	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be
2734	* allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2735	* pRealModeTSS, see hmR3InitFinalizeR0Intel().
2736	*/
2737	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2738	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2739	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2740	{
2741	LogRelFunc(("Invalid real-on-v86 state.\n"));
2742	return VERR_INTERNAL_ERROR;
2743	}
2744
2745	/* Initialize these always, see hmR3InitFinalizeR0().*/
2746	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
2747	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
2748
2749	/* Setup the tagged-TLB flush handlers. */
2750	int rc = hmR0VmxSetupTaggedTlb(pVM);
2751	if (RT_FAILURE(rc))
2752	{
2753	LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2754	return rc;
2755	}
2756
2757	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
2758	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2759	#if HC_ARCH_BITS == 64
2760	if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
2761	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2762	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
2763	{
2764	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2765	}
2766	#endif
2767
2768	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2769	RTCCUINTREG const uHostCR4 = ASMGetCR4();
2770	if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2771	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2772
2773	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2774	{
2775	PVMCPU pVCpu = &pVM->aCpus[i];
2776	AssertPtr(pVCpu);
2777	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2778
2779	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2780	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2781
2782	/* Set revision dword at the beginning of the VMCS structure. */
2783	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
2784
2785	/* Set the VMCS launch state to "clear", see Intel spec. 31.6 "Preparation and launch a virtual machine". */
2786	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2787	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2788	hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2789
2790	/* Load this VMCS as the current VMCS. */
2791	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2792	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2793	hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2794
2795	rc = hmR0VmxSetupPinCtls(pVCpu);
2796	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2797	hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2798
2799	rc = hmR0VmxSetupProcCtls(pVCpu);
2800	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2801	hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2802
2803	rc = hmR0VmxSetupMiscCtls(pVCpu);
2804	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2805	hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2806
2807	rc = hmR0VmxInitXcptBitmap(pVCpu);
2808	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2809	hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2810
2811	#if HC_ARCH_BITS == 32
2812	rc = hmR0VmxInitVmcsReadCache(pVCpu);
2813	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2814	hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2815	#endif
2816
2817	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
2818	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2819	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2820	hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2821
2822	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2823
2824	hmR0VmxUpdateErrorRecord(pVCpu, rc);
2825	}
2826
2827	return VINF_SUCCESS;
2828	}
2829
2830
2831	/**
2832	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2833	* the VMCS.
2834	*
2835	* @returns VBox status code.
2836	*/
2837	static int hmR0VmxExportHostControlRegs(void)
2838	{
2839	RTCCUINTREG uReg = ASMGetCR0();
2840	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2841	AssertRCReturn(rc, rc);
2842
2843	uReg = ASMGetCR3();
2844	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2845	AssertRCReturn(rc, rc);
2846
2847	uReg = ASMGetCR4();
2848	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2849	AssertRCReturn(rc, rc);
2850	return rc;
2851	}
2852
2853
2854	/**
2855	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2856	* the host-state area in the VMCS.
2857	*
2858	* @returns VBox status code.
2859	* @param pVCpu The cross context virtual CPU structure.
2860	*/
2861	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2862	{
2863	#if HC_ARCH_BITS == 64
2864	/**
2865	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2866	* requirements. See hmR0VmxExportHostSegmentRegs().
2867	*/
2868	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2869	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2870	{ \
2871	bool fValidSelector = true; \
2872	if ((selValue) & X86_SEL_LDT) \
2873	{ \
2874	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2875	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2876	} \
2877	if (fValidSelector) \
2878	{ \
2879	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2880	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2881	} \
2882	(selValue) = 0; \
2883	}
2884
2885	/*
2886	* If we've executed guest code using VT-x, the host-state bits will be messed up. We
2887	* should -not- save the messed up state without restoring the original host-state,
2888	* see @bugref{7240}.
2889	*
2890	* This apparently can happen (most likely the FPU changes), deal with it rather than
2891	* asserting. Was observed booting Solaris 10u10 32-bit guest.
2892	*/
2893	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2894	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2895	{
2896	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2897	pVCpu->idCpu));
2898	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2899	}
2900	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2901	#else
2902	RT_NOREF(pVCpu);
2903	#endif
2904
2905	/*
2906	* Host DS, ES, FS and GS segment registers.
2907	*/
2908	#if HC_ARCH_BITS == 64
2909	RTSEL uSelDS = ASMGetDS();
2910	RTSEL uSelES = ASMGetES();
2911	RTSEL uSelFS = ASMGetFS();
2912	RTSEL uSelGS = ASMGetGS();
2913	#else
2914	RTSEL uSelDS = 0;
2915	RTSEL uSelES = 0;
2916	RTSEL uSelFS = 0;
2917	RTSEL uSelGS = 0;
2918	#endif
2919
2920	/*
2921	* Host CS and SS segment registers.
2922	*/
2923	RTSEL uSelCS = ASMGetCS();
2924	RTSEL uSelSS = ASMGetSS();
2925
2926	/*
2927	* Host TR segment register.
2928	*/
2929	RTSEL uSelTR = ASMGetTR();
2930
2931	#if HC_ARCH_BITS == 64
2932	/*
2933	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2934	* gain VM-entry and restore them before we get preempted.
2935	*
2936	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2937	*/
2938	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2939	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2940	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2941	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2942	# undef VMXLOCAL_ADJUST_HOST_SEG
2943	#endif
2944
2945	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2946	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2947	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2948	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2949	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2950	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2951	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2952	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2953	Assert(uSelCS);
2954	Assert(uSelTR);
2955
2956	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2957	#if 0
2958	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE))
2959	Assert(uSelSS != 0);
2960	#endif
2961
2962	/* Write these host selector fields into the host-state area in the VMCS. */
2963	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2964	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2965	#if HC_ARCH_BITS == 64
2966	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2967	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2968	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2969	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2970	#else
2971	NOREF(uSelDS);
2972	NOREF(uSelES);
2973	NOREF(uSelFS);
2974	NOREF(uSelGS);
2975	#endif
2976	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2977	AssertRCReturn(rc, rc);
2978
2979	/*
2980	* Host GDTR and IDTR.
2981	*/
2982	RTGDTR Gdtr;
2983	RTIDTR Idtr;
2984	RT_ZERO(Gdtr);
2985	RT_ZERO(Idtr);
2986	ASMGetGDTR(&Gdtr);
2987	ASMGetIDTR(&Idtr);
2988	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2989	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2990	AssertRCReturn(rc, rc);
2991
2992	#if HC_ARCH_BITS == 64
2993	/*
2994	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
2995	* them to the maximum limit (0xffff) on every VM-exit.
2996	*/
2997	if (Gdtr.cbGdt != 0xffff)
2998	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2999
3000	/*
3001	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
3002	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
3003	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
3004	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
3005	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
3006	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
3007	* at 0xffff on hosts where we are sure it won't cause trouble.
3008	*/
3009	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
3010	if (Idtr.cbIdt < 0x0fff)
3011	# else
3012	if (Idtr.cbIdt != 0xffff)
3013	# endif
3014	{
3015	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
3016	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3017	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3018	}
3019	#endif
3020
3021	/*
3022	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3023	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3024	* RPL should be too in most cases.
3025	*/
3026	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3027	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3028
3029	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3030	#if HC_ARCH_BITS == 64
3031	uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3032
3033	/*
3034	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3035	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3036	* restoration if the host has something else. Task switching is not supported in 64-bit
3037	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3038	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3039	*
3040	* [1] See Intel spec. 3.5 "System Descriptor Types".
3041	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3042	*/
3043	PVM pVM = pVCpu->CTX_SUFF(pVM);
3044	Assert(pDesc->System.u4Type == 11);
3045	if ( pDesc->System.u16LimitLow != 0x67
3046	\|\| pDesc->System.u4LimitHigh)
3047	{
3048	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
3049	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3050	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3051	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
3052	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3053	}
3054
3055	/*
3056	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3057	*/
3058	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
3059	{
3060	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3061	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3062	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3063	{
3064	/* The GDT is read-only but the writable GDT is available. */
3065	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3066	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3067	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3068	AssertRCReturn(rc, rc);
3069	}
3070	}
3071	#else
3072	uintptr_t uTRBase = X86DESC_BASE(pDesc);
3073	#endif
3074	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3075	AssertRCReturn(rc, rc);
3076
3077	/*
3078	* Host FS base and GS base.
3079	*/
3080	#if HC_ARCH_BITS == 64
3081	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3082	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3083	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3084	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3085	AssertRCReturn(rc, rc);
3086
3087	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3088	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3089	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3090	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3091	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3092	#endif
3093	return VINF_SUCCESS;
3094	}
3095
3096
3097	/**
3098	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
3099	* host-state area of the VMCS.
3100	*
3101	* Theses MSRs will be automatically restored on the host after every successful
3102	* VM-exit.
3103	*
3104	* @returns VBox status code.
3105	* @param pVCpu The cross context virtual CPU structure.
3106	*
3107	* @remarks No-long-jump zone!!!
3108	*/
3109	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3110	{
3111	AssertPtr(pVCpu);
3112	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3113
3114	/*
3115	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3116	* rather than swapping them on every VM-entry.
3117	*/
3118	hmR0VmxLazySaveHostMsrs(pVCpu);
3119
3120	/*
3121	* Host Sysenter MSRs.
3122	*/
3123	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3124	#if HC_ARCH_BITS == 32
3125	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3126	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3127	#else
3128	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3129	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3130	#endif
3131	AssertRCReturn(rc, rc);
3132
3133	/*
3134	* Host EFER MSR.
3135	*
3136	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3137	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3138	*/
3139	PVM pVM = pVCpu->CTX_SUFF(pVM);
3140	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3141	{
3142	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3143	AssertRCReturn(rc, rc);
3144	}
3145
3146	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3147
3148	return VINF_SUCCESS;
3149	}
3150
3151
3152	/**
3153	* Figures out if we need to swap the EFER MSR which is particularly expensive.
3154	*
3155	* We check all relevant bits. For now, that's everything besides LMA/LME, as
3156	* these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3157	* hmR0VMxExportGuestEntryCtls().
3158	*
3159	* @returns true if we need to load guest EFER, false otherwise.
3160	* @param pVCpu The cross context virtual CPU structure.
3161	*
3162	* @remarks Requires EFER, CR4.
3163	* @remarks No-long-jump zone!!!
3164	*/
3165	static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu)
3166	{
3167	#ifdef HMVMX_ALWAYS_SWAP_EFER
3168	RT_NOREF(pVCpu);
3169	return true;
3170	#else
3171
3172	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3173	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3174	/* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3175	if (CPUMIsGuestInLongModeEx(pCtx))
3176	return false;
3177	#endif
3178
3179	PVM pVM = pVCpu->CTX_SUFF(pVM);
3180	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3181	uint64_t const u64GuestEfer = pCtx->msrEFER;
3182
3183	/*
3184	* For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3185	* guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3186	*/
3187	if ( CPUMIsGuestInLongModeEx(pCtx)
3188	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3189	{
3190	return true;
3191	}
3192
3193	/*
3194	* If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3195	* affects guest paging. 64-bit paging implies CR4.PAE as well.
3196	* See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3197	*/
3198	if ( (pCtx->cr4 & X86_CR4_PAE)
3199	&& (pCtx->cr0 & X86_CR0_PG)
3200	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3201	{
3202	/* Assert that host is NX capable. */
3203	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
3204	return true;
3205	}
3206
3207	return false;
3208	#endif
3209	}
3210
3211
3212	/**
3213	* Exports the guest state with appropriate VM-entry controls in the VMCS.
3214	*
3215	* These controls can affect things done on VM-exit; e.g. "load debug controls",
3216	* see Intel spec. 24.8.1 "VM-entry controls".
3217	*
3218	* @returns VBox status code.
3219	* @param pVCpu The cross context virtual CPU structure.
3220	*
3221	* @remarks Requires EFER.
3222	* @remarks No-long-jump zone!!!
3223	*/
3224	static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu)
3225	{
3226	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3227	{
3228	PVM pVM = pVCpu->CTX_SUFF(pVM);
3229	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3230	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3231
3232	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3233	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
3234
3235	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3236	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
3237	{
3238	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
3239	Log4Func(("VMX_ENTRY_CTLS_IA32E_MODE_GUEST\n"));
3240	}
3241	else
3242	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
3243
3244	/* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3245	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3246	&& hmR0VmxShouldSwapEferMsr(pVCpu))
3247	{
3248	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
3249	Log4Func(("VMX_ENTRY_CTLS_LOAD_EFER_MSR\n"));
3250	}
3251
3252	/*
3253	* The following should -not- be set (since we're not in SMM mode):
3254	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
3255	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
3256	*/
3257
3258	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
3259	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
3260
3261	if ((fVal & fZap) != fVal)
3262	{
3263	Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3264	pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0, fVal, fZap));
3265	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3266	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3267	}
3268
3269	/* Commit it to the VMCS and update our cache. */
3270	if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3271	{
3272	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3273	AssertRCReturn(rc, rc);
3274	pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3275	}
3276
3277	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3278	}
3279	return VINF_SUCCESS;
3280	}
3281
3282
3283	/**
3284	* Exports the guest state with appropriate VM-exit controls in the VMCS.
3285	*
3286	* @returns VBox status code.
3287	* @param pVCpu The cross context virtual CPU structure.
3288	*
3289	* @remarks Requires EFER.
3290	*/
3291	static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu)
3292	{
3293	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3294	{
3295	PVM pVM = pVCpu->CTX_SUFF(pVM);
3296	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3297	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3298
3299	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3300	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
3301
3302	/*
3303	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3304	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3305	* hmR0VmxExportHostMsrs().
3306	*/
3307	#if HC_ARCH_BITS == 64
3308	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
3309	Log4Func(("VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE\n"));
3310	#else
3311	Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3312	\|\| pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3313	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3314	if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3315	{
3316	/* The switcher returns to long mode, EFER is managed by the switcher. */
3317	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
3318	Log4Func(("VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE\n"));
3319	}
3320	else
3321	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
3322	#endif
3323
3324	/* If the newer VMCS fields for managing EFER exists, use it. */
3325	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3326	&& hmR0VmxShouldSwapEferMsr(pVCpu))
3327	{
3328	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
3329	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
3330	Log4Func(("VMX_EXIT_CTLS_SAVE_EFER_MSR and VMX_EXIT_CTLS_LOAD_EFER_MSR\n"));
3331	}
3332
3333	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3334	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
3335
3336	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
3337	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
3338	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
3339
3340	/* Enable saving of the VMX preemption timer value on VM-exit. */
3341	if ( pVM->hm.s.vmx.fUsePreemptTimer
3342	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
3343	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
3344
3345	if ((fVal & fZap) != fVal)
3346	{
3347	LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3348	pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0, fVal, fZap));
3349	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3350	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3351	}
3352
3353	/* Commit it to the VMCS and update our cache. */
3354	if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3355	{
3356	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3357	AssertRCReturn(rc, rc);
3358	pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3359	}
3360
3361	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3362	}
3363	return VINF_SUCCESS;
3364	}
3365
3366
3367	/**
3368	* Sets the TPR threshold in the VMCS.
3369	*
3370	* @returns VBox status code.
3371	* @param pVCpu The cross context virtual CPU structure.
3372	* @param u32TprThreshold The TPR threshold (task-priority class only).
3373	*/
3374	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3375	{
3376	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
3377	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3378	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3379	}
3380
3381
3382	/**
3383	* Exports the guest APIC TPR state into the VMCS.
3384	*
3385	* @returns VBox status code.
3386	* @param pVCpu The cross context virtual CPU structure.
3387	*
3388	* @remarks No-long-jump zone!!!
3389	*/
3390	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3391	{
3392	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3393	{
3394	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
3395
3396	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3397	&& APICIsEnabled(pVCpu))
3398	{
3399	/*
3400	* Setup TPR shadowing.
3401	*/
3402	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
3403	{
3404	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3405
3406	bool fPendingIntr = false;
3407	uint8_t u8Tpr = 0;
3408	uint8_t u8PendingIntr = 0;
3409	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3410	AssertRCReturn(rc, rc);
3411
3412	/*
3413	* If there are interrupts pending but masked by the TPR, instruct VT-x to
3414	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3415	* priority of the pending interrupt so we can deliver the interrupt. If there
3416	* are no interrupts pending, set threshold to 0 to not cause any
3417	* TPR-below-threshold VM-exits.
3418	*/
3419	pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3420	uint32_t u32TprThreshold = 0;
3421	if (fPendingIntr)
3422	{
3423	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3424	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3425	const uint8_t u8TprPriority = u8Tpr >> 4;
3426	if (u8PendingPriority <= u8TprPriority)
3427	u32TprThreshold = u8PendingPriority;
3428	}
3429
3430	rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3431	AssertRCReturn(rc, rc);
3432	}
3433	}
3434	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3435	}
3436	return VINF_SUCCESS;
3437	}
3438
3439
3440	/**
3441	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3442	*
3443	* @returns Guest's interruptibility-state.
3444	* @param pVCpu The cross context virtual CPU structure.
3445	*
3446	* @remarks No-long-jump zone!!!
3447	*/
3448	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu)
3449	{
3450	/*
3451	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3452	*/
3453	uint32_t fIntrState = 0;
3454	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3455	{
3456	/* If inhibition is active, RIP & RFLAGS should've been accessed
3457	(i.e. read previously from the VMCS or from ring-3). */
3458	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3459	#ifdef VBOX_STRICT
3460	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
3461	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3462	#endif
3463	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3464	{
3465	if (pCtx->eflags.Bits.u1IF)
3466	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
3467	else
3468	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
3469	}
3470	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3471	{
3472	/*
3473	* We can clear the inhibit force flag as even if we go back to the recompiler
3474	* without executing guest code in VT-x, the flag's condition to be cleared is
3475	* met and thus the cleared state is correct.
3476	*/
3477	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3478	}
3479	}
3480
3481	/*
3482	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3483	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3484	* setting this would block host-NMIs and IRET will not clear the blocking.
3485	*
3486	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3487	*/
3488	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3489	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
3490	{
3491	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
3492	}
3493
3494	return fIntrState;
3495	}
3496
3497
3498	/**
3499	* Exports the guest's interruptibility-state into the guest-state area in the
3500	* VMCS.
3501	*
3502	* @returns VBox status code.
3503	* @param pVCpu The cross context virtual CPU structure.
3504	* @param fIntrState The interruptibility-state to set.
3505	*/
3506	static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3507	{
3508	NOREF(pVCpu);
3509	AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3510	Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3511	return VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
3512	}
3513
3514
3515	/**
3516	* Exports the exception intercepts required for guest execution in the VMCS.
3517	*
3518	* @returns VBox status code.
3519	* @param pVCpu The cross context virtual CPU structure.
3520	*
3521	* @remarks No-long-jump zone!!!
3522	*/
3523	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3524	{
3525	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3526	{
3527	uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3528
3529	/* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3530	if (pVCpu->hm.s.fGIMTrapXcptUD)
3531	uXcptBitmap \|= RT_BIT(X86_XCPT_UD);
3532	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3533	else
3534	uXcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3535	#endif
3536
3537	Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_AC));
3538	Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_DB));
3539
3540	if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3541	{
3542	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3543	AssertRCReturn(rc, rc);
3544	pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3545	}
3546
3547	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3548	Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", uXcptBitmap));
3549	}
3550	return VINF_SUCCESS;
3551	}
3552
3553
3554	/**
3555	* Exports the guest's RIP into the guest-state area in the VMCS.
3556	*
3557	* @returns VBox status code.
3558	* @param pVCpu The cross context virtual CPU structure.
3559	*
3560	* @remarks No-long-jump zone!!!
3561	*/
3562	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
3563	{
3564	int rc = VINF_SUCCESS;
3565	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3566	{
3567	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
3568
3569	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
3570	AssertRCReturn(rc, rc);
3571
3572	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3573	Log4Func(("RIP=%#RX64\n", pVCpu->cpum.GstCtx.rip));
3574	}
3575	return rc;
3576	}
3577
3578
3579	/**
3580	* Exports the guest's RSP into the guest-state area in the VMCS.
3581	*
3582	* @returns VBox status code.
3583	* @param pVCpu The cross context virtual CPU structure.
3584	*
3585	* @remarks No-long-jump zone!!!
3586	*/
3587	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
3588	{
3589	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3590	{
3591	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
3592
3593	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
3594	AssertRCReturn(rc, rc);
3595
3596	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3597	}
3598	return VINF_SUCCESS;
3599	}
3600
3601
3602	/**
3603	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
3604	*
3605	* @returns VBox status code.
3606	* @param pVCpu The cross context virtual CPU structure.
3607	*
3608	* @remarks No-long-jump zone!!!
3609	*/
3610	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu)
3611	{
3612	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3613	{
3614	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
3615
3616	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3617	Let us assert it as such and use 32-bit VMWRITE. */
3618	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
3619	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
3620	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3621	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
3622
3623	/*
3624	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3625	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3626	* can run the real-mode guest code under Virtual 8086 mode.
3627	*/
3628	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3629	{
3630	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3631	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3632	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3633	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3634	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3635	}
3636
3637	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3638	AssertRCReturn(rc, rc);
3639
3640	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3641	Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3642	}
3643	return VINF_SUCCESS;
3644	}
3645
3646
3647	/**
3648	* Exports the guest CR0 control register into the guest-state area in the VMCS.
3649	*
3650	* The guest FPU state is always pre-loaded hence we don't need to bother about
3651	* sharing FPU related CR0 bits between the guest and host.
3652	*
3653	* @returns VBox status code.
3654	* @param pVCpu The cross context virtual CPU structure.
3655	*
3656	* @remarks No-long-jump zone!!!
3657	*/
3658	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu)
3659	{
3660	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3661	{
3662	PVM pVM = pVCpu->CTX_SUFF(pVM);
3663	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3664	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.cr0));
3665
3666	uint32_t const u32ShadowCr0 = pVCpu->cpum.GstCtx.cr0;
3667	uint32_t u32GuestCr0 = pVCpu->cpum.GstCtx.cr0;
3668
3669	/*
3670	* Setup VT-x's view of the guest CR0.
3671	* Minimize VM-exits due to CR3 changes when we have NestedPaging.
3672	*/
3673	uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3674	if (pVM->hm.s.fNestedPaging)
3675	{
3676	if (CPUMIsGuestPagingEnabled(pVCpu))
3677	{
3678	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3679	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
3680	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
3681	}
3682	else
3683	{
3684	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3685	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
3686	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3687	}
3688
3689	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3690	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3691	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
3692	}
3693	else
3694	{
3695	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3696	u32GuestCr0 \|= X86_CR0_WP;
3697	}
3698
3699	/*
3700	* Guest FPU bits.
3701	*
3702	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3703	* using CR0.TS.
3704	*
3705	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3706	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3707	*/
3708	u32GuestCr0 \|= X86_CR0_NE;
3709
3710	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3711	bool const fInterceptMF = !(u32ShadowCr0 & X86_CR0_NE);
3712
3713	/*
3714	* Update exception intercepts.
3715	*/
3716	uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3717	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3718	{
3719	Assert(PDMVmmDevHeapIsEnabled(pVM));
3720	Assert(pVM->hm.s.vmx.pRealModeTSS);
3721	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3722	}
3723	else
3724	{
3725	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3726	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3727	if (fInterceptMF)
3728	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
3729	}
3730
3731	/* Additional intercepts for debugging, define these yourself explicitly. */
3732	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3733	uXcptBitmap \|= 0
3734	\| RT_BIT(X86_XCPT_BP)
3735	\| RT_BIT(X86_XCPT_DE)
3736	\| RT_BIT(X86_XCPT_NM)
3737	\| RT_BIT(X86_XCPT_TS)
3738	\| RT_BIT(X86_XCPT_UD)
3739	\| RT_BIT(X86_XCPT_NP)
3740	\| RT_BIT(X86_XCPT_SS)
3741	\| RT_BIT(X86_XCPT_GP)
3742	\| RT_BIT(X86_XCPT_PF)
3743	\| RT_BIT(X86_XCPT_MF)
3744	;
3745	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3746	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
3747	#endif
3748	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
3749
3750	/*
3751	* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3752	*/
3753	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3754	uint32_t fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3755	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3756	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3757	else
3758	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3759
3760	u32GuestCr0 \|= fSetCr0;
3761	u32GuestCr0 &= fZapCr0;
3762	u32GuestCr0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3763
3764	/*
3765	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3766	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3767	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3768	*/
3769	uint32_t u32Cr0Mask = X86_CR0_PE
3770	\| X86_CR0_NE
3771	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3772	\| X86_CR0_PG
3773	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3774	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3775	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3776
3777	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3778	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3779	* and @bugref{6944}. */
3780	#if 0
3781	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3782	u32Cr0Mask &= ~X86_CR0_PE;
3783	#endif
3784	/*
3785	* Finally, update VMCS fields with the CR0 values and the exception bitmap.
3786	*/
3787	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCr0);
3788	rc \|= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32ShadowCr0);
3789	if (u32Cr0Mask != pVCpu->hm.s.vmx.u32Cr0Mask)
3790	rc \|= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32Cr0Mask);
3791	if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3792	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3793	if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3794	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3795	AssertRCReturn(rc, rc);
3796
3797	/* Update our caches. */
3798	pVCpu->hm.s.vmx.u32Cr0Mask = u32Cr0Mask;
3799	pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3800	pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3801
3802	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3803
3804	Log4Func(("u32Cr0Mask=%#RX32 u32ShadowCr0=%#RX32 u32GuestCr0=%#RX32 (fSetCr0=%#RX32 fZapCr0=%#RX32\n", u32Cr0Mask,
3805	u32ShadowCr0, u32GuestCr0, fSetCr0, fZapCr0));
3806	}
3807
3808	return VINF_SUCCESS;
3809	}
3810
3811
3812	/**
3813	* Exports the guest control registers (CR3, CR4) into the guest-state area
3814	* in the VMCS.
3815	*
3816	* @returns VBox strict status code.
3817	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3818	* without unrestricted guest access and the VMMDev is not presently
3819	* mapped (e.g. EFI32).
3820	*
3821	* @param pVCpu The cross context virtual CPU structure.
3822	*
3823	* @remarks No-long-jump zone!!!
3824	*/
3825	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu)
3826	{
3827	int rc = VINF_SUCCESS;
3828	PVM pVM = pVCpu->CTX_SUFF(pVM);
3829
3830	/*
3831	* Guest CR2.
3832	* It's always loaded in the assembler code. Nothing to do here.
3833	*/
3834
3835	/*
3836	* Guest CR3.
3837	*/
3838	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3839	{
3840	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
3841
3842	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3843	if (pVM->hm.s.fNestedPaging)
3844	{
3845	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3846
3847	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3848	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3849	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3850	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3851
3852	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3853	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3854	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3855
3856	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3857	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3858	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3859	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3860	AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3861	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3862	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3863
3864	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3865	AssertRCReturn(rc, rc);
3866
3867	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3868	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3869	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
3870	{
3871	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3872	if (CPUMIsGuestInPAEModeEx(pCtx))
3873	{
3874	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3875	AssertRCReturn(rc, rc);
3876	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3877	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3878	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3879	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3880	AssertRCReturn(rc, rc);
3881	}
3882
3883	/*
3884	* The guest's view of its CR3 is unblemished with Nested Paging when the
3885	* guest is using paging or we have unrestricted guest execution to handle
3886	* the guest when it's not using paging.
3887	*/
3888	GCPhysGuestCR3 = pCtx->cr3;
3889	}
3890	else
3891	{
3892	/*
3893	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
3894	* thinks it accesses physical memory directly, we use our identity-mapped
3895	* page table to map guest-linear to guest-physical addresses. EPT takes care
3896	* of translating it to host-physical addresses.
3897	*/
3898	RTGCPHYS GCPhys;
3899	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3900
3901	/* We obtain it here every time as the guest could have relocated this PCI region. */
3902	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3903	if (RT_SUCCESS(rc))
3904	{ /* likely */ }
3905	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3906	{
3907	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3908	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3909	}
3910	else
3911	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3912
3913	GCPhysGuestCR3 = GCPhys;
3914	}
3915
3916	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
3917	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3918	AssertRCReturn(rc, rc);
3919	}
3920	else
3921	{
3922	/* Non-nested paging case, just use the hypervisor's CR3. */
3923	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3924
3925	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
3926	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3927	AssertRCReturn(rc, rc);
3928	}
3929
3930	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3931	}
3932
3933	/*
3934	* Guest CR4.
3935	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
3936	*/
3937	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3938	{
3939	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3940	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
3941	Assert(!RT_HI_U32(pCtx->cr4));
3942
3943	uint32_t u32GuestCr4 = pCtx->cr4;
3944	uint32_t const u32ShadowCr4 = pCtx->cr4;
3945
3946	/*
3947	* Setup VT-x's view of the guest CR4.
3948	*
3949	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3950	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3951	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3952	*
3953	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3954	*/
3955	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3956	{
3957	Assert(pVM->hm.s.vmx.pRealModeTSS);
3958	Assert(PDMVmmDevHeapIsEnabled(pVM));
3959	u32GuestCr4 &= ~X86_CR4_VME;
3960	}
3961
3962	if (pVM->hm.s.fNestedPaging)
3963	{
3964	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3965	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3966	{
3967	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3968	u32GuestCr4 \|= X86_CR4_PSE;
3969	/* Our identity mapping is a 32-bit page directory. */
3970	u32GuestCr4 &= ~X86_CR4_PAE;
3971	}
3972	/* else use guest CR4.*/
3973	}
3974	else
3975	{
3976	/*
3977	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3978	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3979	*/
3980	switch (pVCpu->hm.s.enmShadowMode)
3981	{
3982	case PGMMODE_REAL: /* Real-mode. */
3983	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3984	case PGMMODE_32_BIT: /* 32-bit paging. */
3985	{
3986	u32GuestCr4 &= ~X86_CR4_PAE;
3987	break;
3988	}
3989
3990	case PGMMODE_PAE: /* PAE paging. */
3991	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3992	{
3993	u32GuestCr4 \|= X86_CR4_PAE;
3994	break;
3995	}
3996
3997	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3998	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3999	#ifdef VBOX_ENABLE_64_BITS_GUESTS
4000	break;
4001	#endif
4002	default:
4003	AssertFailed();
4004	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4005	}
4006	}
4007
4008	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4009	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4010	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4011	u32GuestCr4 \|= fSetCr4;
4012	u32GuestCr4 &= fZapCr4;
4013
4014	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
4015	that would cause a VM-exit. */
4016	uint32_t u32Cr4Mask = X86_CR4_VME
4017	\| X86_CR4_PAE
4018	\| X86_CR4_PGE
4019	\| X86_CR4_PSE
4020	\| X86_CR4_VMXE;
4021	if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4022	u32Cr4Mask \|= X86_CR4_OSXSAVE;
4023	if (pVM->cpum.ro.GuestFeatures.fPcid)
4024	u32Cr4Mask \|= X86_CR4_PCIDE;
4025
4026	/* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4027	into the VMCS and update our cache. */
4028	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCr4);
4029	rc \|= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32ShadowCr4);
4030	if (pVCpu->hm.s.vmx.u32Cr4Mask != u32Cr4Mask)
4031	rc \|= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32Cr4Mask);
4032	AssertRCReturn(rc, rc);
4033	pVCpu->hm.s.vmx.u32Cr4Mask = u32Cr4Mask;
4034
4035	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4036	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
4037
4038	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4039
4040	Log4Func(("u32GuestCr4=%#RX32 u32ShadowCr4=%#RX32 (fSetCr4=%#RX32 fZapCr4=%#RX32)\n", u32GuestCr4, u32ShadowCr4, fSetCr4,
4041	fZapCr4));
4042	}
4043	return rc;
4044	}
4045
4046
4047	/**
4048	* Exports the guest debug registers into the guest-state area in the VMCS.
4049	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4050	*
4051	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4052	*
4053	* @returns VBox status code.
4054	* @param pVCpu The cross context virtual CPU structure.
4055	*
4056	* @remarks No-long-jump zone!!!
4057	*/
4058	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu)
4059	{
4060	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4061
4062	#ifdef VBOX_STRICT
4063	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4064	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
4065	{
4066	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4067	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
4068	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
4069	}
4070	#endif
4071
4072	bool fSteppingDB = false;
4073	bool fInterceptMovDRx = false;
4074	uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4075	if (pVCpu->hm.s.fSingleInstruction)
4076	{
4077	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4078	PVM pVM = pVCpu->CTX_SUFF(pVM);
4079	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
4080	{
4081	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
4082	Assert(fSteppingDB == false);
4083	}
4084	else
4085	{
4086	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
4087	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
4088	pVCpu->hm.s.fClearTrapFlag = true;
4089	fSteppingDB = true;
4090	}
4091	}
4092
4093	uint32_t u32GuestDr7;
4094	if ( fSteppingDB
4095	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4096	{
4097	/*
4098	* Use the combined guest and host DRx values found in the hypervisor register set
4099	* because the debugger has breakpoints active or someone is single stepping on the
4100	* host side without a monitor trap flag.
4101	*
4102	* Note! DBGF expects a clean DR6 state before executing guest code.
4103	*/
4104	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4105	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4106	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
4107	{
4108	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4109	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4110	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4111	}
4112	else
4113	#endif
4114	if (!CPUMIsHyperDebugStateActive(pVCpu))
4115	{
4116	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4117	Assert(CPUMIsHyperDebugStateActive(pVCpu));
4118	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4119	}
4120
4121	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4122	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4123	pVCpu->hm.s.fUsingHyperDR7 = true;
4124	fInterceptMovDRx = true;
4125	}
4126	else
4127	{
4128	/*
4129	* If the guest has enabled debug registers, we need to load them prior to
4130	* executing guest code so they'll trigger at the right time.
4131	*/
4132	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
4133	{
4134	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4135	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4136	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
4137	{
4138	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4139	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4140	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4141	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4142	}
4143	else
4144	#endif
4145	if (!CPUMIsGuestDebugStateActive(pVCpu))
4146	{
4147	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4148	Assert(CPUMIsGuestDebugStateActive(pVCpu));
4149	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4150	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4151	}
4152	Assert(!fInterceptMovDRx);
4153	}
4154	/*
4155	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4156	* must intercept #DB in order to maintain a correct DR6 guest value, and
4157	* because we need to intercept it to prevent nested #DBs from hanging the
4158	* CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4159	*/
4160	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4161	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4162	&& !CPUMIsGuestDebugStateActive(pVCpu))
4163	#else
4164	else if (!CPUMIsGuestDebugStateActive(pVCpu))
4165	#endif
4166	{
4167	fInterceptMovDRx = true;
4168	}
4169
4170	/* Update DR7 with the actual guest value. */
4171	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
4172	pVCpu->hm.s.fUsingHyperDR7 = false;
4173	}
4174
4175	if (fInterceptMovDRx)
4176	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
4177	else
4178	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
4179
4180	/*
4181	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4182	* monitor-trap flag and update our cache.
4183	*/
4184	if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4185	{
4186	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4187	AssertRCReturn(rc2, rc2);
4188	pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4189	}
4190
4191	/*
4192	* Update guest DR7.
4193	*/
4194	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
4195	AssertRCReturn(rc, rc);
4196
4197	return VINF_SUCCESS;
4198	}
4199
4200
4201	#ifdef VBOX_STRICT
4202	/**
4203	* Strict function to validate segment registers.
4204	*
4205	* @param pVCpu The cross context virtual CPU structure.
4206	*
4207	* @remarks Will import guest CR0 on strict builds during validation of
4208	* segments.
4209	*/
4210	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu)
4211	{
4212	/*
4213	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4214	*
4215	* The reason we check for attribute value 0 in this function and not just the unusable bit is
4216	* because hmR0VmxExportGuestSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4217	* and doesn't change the guest-context value.
4218	*/
4219	PVM pVM = pVCpu->CTX_SUFF(pVM);
4220	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4221	hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4222	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4223	&& ( !CPUMIsGuestInRealModeEx(pCtx)
4224	&& !CPUMIsGuestInV86ModeEx(pCtx)))
4225	{
4226	/* Protected mode checks */
4227	/* CS */
4228	Assert(pCtx->cs.Attr.n.u1Present);
4229	Assert(!(pCtx->cs.Attr.u & 0xf00));
4230	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4231	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4232	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4233	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4234	\|\| (pCtx->cs.Attr.n.u1Granularity));
4235	/* CS cannot be loaded with NULL in protected mode. */
4236	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4237	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4238	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4239	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4240	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4241	else
4242	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4243	/* SS */
4244	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4245	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4246	if ( !(pCtx->cr0 & X86_CR0_PE)
4247	\|\| pCtx->cs.Attr.n.u4Type == 3)
4248	{
4249	Assert(!pCtx->ss.Attr.n.u2Dpl);
4250	}
4251	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4252	{
4253	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4254	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4255	Assert(pCtx->ss.Attr.n.u1Present);
4256	Assert(!(pCtx->ss.Attr.u & 0xf00));
4257	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4258	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4259	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4260	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4261	\|\| (pCtx->ss.Attr.n.u1Granularity));
4262	}
4263	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmentReg(). */
4264	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4265	{
4266	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4267	Assert(pCtx->ds.Attr.n.u1Present);
4268	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4269	Assert(!(pCtx->ds.Attr.u & 0xf00));
4270	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4271	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4272	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4273	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4274	\|\| (pCtx->ds.Attr.n.u1Granularity));
4275	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4276	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4277	}
4278	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4279	{
4280	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4281	Assert(pCtx->es.Attr.n.u1Present);
4282	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4283	Assert(!(pCtx->es.Attr.u & 0xf00));
4284	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4285	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4286	\|\| !(pCtx->es.Attr.n.u1Granularity));
4287	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4288	\|\| (pCtx->es.Attr.n.u1Granularity));
4289	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4290	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4291	}
4292	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4293	{
4294	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4295	Assert(pCtx->fs.Attr.n.u1Present);
4296	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4297	Assert(!(pCtx->fs.Attr.u & 0xf00));
4298	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4299	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4300	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4301	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4302	\|\| (pCtx->fs.Attr.n.u1Granularity));
4303	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4304	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4305	}
4306	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4307	{
4308	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4309	Assert(pCtx->gs.Attr.n.u1Present);
4310	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4311	Assert(!(pCtx->gs.Attr.u & 0xf00));
4312	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4313	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4314	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4315	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4316	\|\| (pCtx->gs.Attr.n.u1Granularity));
4317	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4318	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4319	}
4320	/* 64-bit capable CPUs. */
4321	# if HC_ARCH_BITS == 64
4322	Assert(!RT_HI_U32(pCtx->cs.u64Base));
4323	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
4324	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
4325	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
4326	# endif
4327	}
4328	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4329	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4330	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4331	{
4332	/* Real and v86 mode checks. */
4333	/* hmR0VmxExportGuestSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4334	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4335	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4336	{
4337	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4338	}
4339	else
4340	{
4341	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4342	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4343	}
4344
4345	/* CS */
4346	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4347	Assert(pCtx->cs.u32Limit == 0xffff);
4348	Assert(u32CSAttr == 0xf3);
4349	/* SS */
4350	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4351	Assert(pCtx->ss.u32Limit == 0xffff);
4352	Assert(u32SSAttr == 0xf3);
4353	/* DS */
4354	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4355	Assert(pCtx->ds.u32Limit == 0xffff);
4356	Assert(u32DSAttr == 0xf3);
4357	/* ES */
4358	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4359	Assert(pCtx->es.u32Limit == 0xffff);
4360	Assert(u32ESAttr == 0xf3);
4361	/* FS */
4362	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4363	Assert(pCtx->fs.u32Limit == 0xffff);
4364	Assert(u32FSAttr == 0xf3);
4365	/* GS */
4366	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4367	Assert(pCtx->gs.u32Limit == 0xffff);
4368	Assert(u32GSAttr == 0xf3);
4369	/* 64-bit capable CPUs. */
4370	# if HC_ARCH_BITS == 64
4371	Assert(!RT_HI_U32(pCtx->cs.u64Base));
4372	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
4373	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
4374	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
4375	# endif
4376	}
4377	}
4378	#endif /* VBOX_STRICT */
4379
4380
4381	/**
4382	* Exports a guest segment register into the guest-state area in the VMCS.
4383	*
4384	* @returns VBox status code.
4385	* @param pVCpu The cross context virtual CPU structure.
4386	* @param idxSel Index of the selector in the VMCS.
4387	* @param idxLimit Index of the segment limit in the VMCS.
4388	* @param idxBase Index of the segment base in the VMCS.
4389	* @param idxAccess Index of the access rights of the segment in the VMCS.
4390	* @param pSelReg Pointer to the segment selector.
4391	*
4392	* @remarks No-long-jump zone!!!
4393	*/
4394	static int hmR0VmxExportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
4395	PCCPUMSELREG pSelReg)
4396	{
4397	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4398	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4399	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4400	AssertRCReturn(rc, rc);
4401
4402	uint32_t u32Access = pSelReg->Attr.u;
4403	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4404	{
4405	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4406	u32Access = 0xf3;
4407	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4408	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4409	}
4410	else
4411	{
4412	/*
4413	* The way to differentiate between whether this is really a null selector or was just
4414	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
4415	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4416	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4417	* NULL selectors loaded in protected-mode have their attribute as 0.
4418	*/
4419	if (!u32Access)
4420	u32Access = X86DESCATTR_UNUSABLE;
4421	}
4422
4423	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4424	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4425	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4426
4427	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4428	AssertRCReturn(rc, rc);
4429	return rc;
4430	}
4431
4432
4433	/**
4434	* Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4435	* into the guest-state area in the VMCS.
4436	*
4437	* @returns VBox status code.
4438	* @param pVCpu The cross context virtual CPU structure.
4439	*
4440	* @remarks Will import guest CR0 on strict builds during validation of
4441	* segments.
4442	* @remarks No-long-jump zone!!!
4443	*/
4444	static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu)
4445	{
4446	int rc = VERR_INTERNAL_ERROR_5;
4447	PVM pVM = pVCpu->CTX_SUFF(pVM);
4448	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4449
4450	/*
4451	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4452	*/
4453	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4454	{
4455	#ifdef VBOX_WITH_REM
4456	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4457	{
4458	Assert(pVM->hm.s.vmx.pRealModeTSS);
4459	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4460	if ( pVCpu->hm.s.vmx.fWasInRealMode
4461	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4462	{
4463	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4464	in real-mode (e.g. OpenBSD 4.0) */
4465	REMFlushTBs(pVM);
4466	Log4Func(("Switch to protected mode detected!\n"));
4467	pVCpu->hm.s.vmx.fWasInRealMode = false;
4468	}
4469	}
4470	#endif
4471	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
4472	{
4473	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
4474	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4475	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pCtx->cs.Attr.u;
4476	rc = HMVMX_EXPORT_SREG(CS, &pCtx->cs);
4477	AssertRCReturn(rc, rc);
4478	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
4479	}
4480
4481	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
4482	{
4483	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
4484	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4485	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pCtx->ss.Attr.u;
4486	rc = HMVMX_EXPORT_SREG(SS, &pCtx->ss);
4487	AssertRCReturn(rc, rc);
4488	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
4489	}
4490
4491	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
4492	{
4493	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
4494	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4495	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pCtx->ds.Attr.u;
4496	rc = HMVMX_EXPORT_SREG(DS, &pCtx->ds);
4497	AssertRCReturn(rc, rc);
4498	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
4499	}
4500
4501	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
4502	{
4503	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
4504	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4505	pVCpu->hm.s.vmx.RealMode.AttrES.u = pCtx->es.Attr.u;
4506	rc = HMVMX_EXPORT_SREG(ES, &pCtx->es);
4507	AssertRCReturn(rc, rc);
4508	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
4509	}
4510
4511	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
4512	{
4513	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
4514	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4515	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pCtx->fs.Attr.u;
4516	rc = HMVMX_EXPORT_SREG(FS, &pCtx->fs);
4517	AssertRCReturn(rc, rc);
4518	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
4519	}
4520
4521	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
4522	{
4523	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
4524	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4525	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pCtx->gs.Attr.u;
4526	rc = HMVMX_EXPORT_SREG(GS, &pCtx->gs);
4527	AssertRCReturn(rc, rc);
4528	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
4529	}
4530
4531	#ifdef VBOX_STRICT
4532	hmR0VmxValidateSegmentRegs(pVCpu);
4533	#endif
4534
4535	Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pCtx->cs.Sel, pCtx->cs.u64Base,
4536	pCtx->cs.u32Limit, pCtx->cs.Attr.u));
4537	}
4538
4539	/*
4540	* Guest TR.
4541	*/
4542	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4543	{
4544	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
4545
4546	/*
4547	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4548	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4549	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4550	*/
4551	uint16_t u16Sel = 0;
4552	uint32_t u32Limit = 0;
4553	uint64_t u64Base = 0;
4554	uint32_t u32AccessRights = 0;
4555
4556	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4557	{
4558	u16Sel = pCtx->tr.Sel;
4559	u32Limit = pCtx->tr.u32Limit;
4560	u64Base = pCtx->tr.u64Base;
4561	u32AccessRights = pCtx->tr.Attr.u;
4562	}
4563	else
4564	{
4565	Assert(pVM->hm.s.vmx.pRealModeTSS);
4566	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
4567
4568	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4569	RTGCPHYS GCPhys;
4570	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4571	AssertRCReturn(rc, rc);
4572
4573	X86DESCATTR DescAttr;
4574	DescAttr.u = 0;
4575	DescAttr.n.u1Present = 1;
4576	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4577
4578	u16Sel = 0;
4579	u32Limit = HM_VTX_TSS_SIZE;
4580	u64Base = GCPhys; /* in real-mode phys = virt. */
4581	u32AccessRights = DescAttr.u;
4582	}
4583
4584	/* Validate. */
4585	Assert(!(u16Sel & RT_BIT(2)));
4586	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4587	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4588	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4589	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4590	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4591	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4592	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4593	Assert( (u32Limit & 0xfff) == 0xfff
4594	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4595	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
4596	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4597
4598	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4599	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4600	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4601	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4602	AssertRCReturn(rc, rc);
4603
4604	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4605	Log4Func(("TR base=%#RX64\n", pCtx->tr.u64Base));
4606	}
4607
4608	/*
4609	* Guest GDTR.
4610	*/
4611	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4612	{
4613	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
4614
4615	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
4616	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
4617	AssertRCReturn(rc, rc);
4618
4619	/* Validate. */
4620	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4621
4622	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4623	Log4Func(("GDTR base=%#RX64\n", pCtx->gdtr.pGdt));
4624	}
4625
4626	/*
4627	* Guest LDTR.
4628	*/
4629	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4630	{
4631	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
4632
4633	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4634	uint32_t u32Access = 0;
4635	if (!pCtx->ldtr.Attr.u)
4636	u32Access = X86DESCATTR_UNUSABLE;
4637	else
4638	u32Access = pCtx->ldtr.Attr.u;
4639
4640	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
4641	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
4642	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4643	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
4644	AssertRCReturn(rc, rc);
4645
4646	/* Validate. */
4647	if (!(u32Access & X86DESCATTR_UNUSABLE))
4648	{
4649	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4650	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4651	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4652	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4653	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4654	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4655	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
4656	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4657	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
4658	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4659	}
4660
4661	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4662	Log4Func(("LDTR base=%#RX64\n", pCtx->ldtr.u64Base));
4663	}
4664
4665	/*
4666	* Guest IDTR.
4667	*/
4668	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4669	{
4670	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
4671
4672	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
4673	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
4674	AssertRCReturn(rc, rc);
4675
4676	/* Validate. */
4677	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4678
4679	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4680	Log4Func(("IDTR base=%#RX64\n", pCtx->idtr.pIdt));
4681	}
4682
4683	return VINF_SUCCESS;
4684	}
4685
4686
4687	/**
4688	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4689	* areas.
4690	*
4691	* These MSRs will automatically be loaded to the host CPU on every successful
4692	* VM-entry and stored from the host CPU on every successful VM-exit. This also
4693	* creates/updates MSR slots for the host MSRs. The actual host MSR values are
4694	* -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4695	*
4696	* Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4697	*
4698	* @returns VBox status code.
4699	* @param pVCpu The cross context virtual CPU structure.
4700	*
4701	* @remarks No-long-jump zone!!!
4702	*/
4703	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu)
4704	{
4705	AssertPtr(pVCpu);
4706	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4707
4708	/*
4709	* MSRs that we use the auto-load/store MSR area in the VMCS.
4710	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4711	*/
4712	PVM pVM = pVCpu->CTX_SUFF(pVM);
4713	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4714	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4715	{
4716	if (pVM->hm.s.fAllow64BitGuests)
4717	{
4718	#if HC_ARCH_BITS == 32
4719	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
4720
4721	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pCtx->msrLSTAR, false, NULL);
4722	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pCtx->msrSTAR, false, NULL);
4723	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pCtx->msrSFMASK, false, NULL);
4724	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, false, NULL);
4725	AssertRCReturn(rc, rc);
4726	# ifdef LOG_ENABLED
4727	PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4728	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4729	Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4730	# endif
4731	#endif
4732	}
4733	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4734	}
4735
4736	/*
4737	* Guest Sysenter MSRs.
4738	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4739	* VM-exits on WRMSRs for these MSRs.
4740	*/
4741	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4742	{
4743	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4744
4745	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4746	{
4747	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
4748	AssertRCReturn(rc, rc);
4749	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4750	}
4751
4752	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4753	{
4754	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
4755	AssertRCReturn(rc, rc);
4756	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4757	}
4758
4759	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4760	{
4761	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
4762	AssertRCReturn(rc, rc);
4763	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4764	}
4765	}
4766
4767	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4768	{
4769	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
4770
4771	if (hmR0VmxShouldSwapEferMsr(pVCpu))
4772	{
4773	/*
4774	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4775	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4776	*/
4777	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4778	{
4779	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
4780	AssertRCReturn(rc,rc);
4781	Log4Func(("EFER=%#RX64\n", pCtx->msrEFER));
4782	}
4783	else
4784	{
4785	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pCtx->msrEFER, false /* fUpdateHostMsr */,
4786	NULL /* pfAddedAndUpdated */);
4787	AssertRCReturn(rc, rc);
4788
4789	/* We need to intercept reads too, see @bugref{7386#c16}. */
4790	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
4791	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4792	Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pCtx->msrEFER,
4793	pVCpu->hm.s.vmx.cMsrs));
4794	}
4795	}
4796	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4797	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4798	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4799	}
4800
4801	return VINF_SUCCESS;
4802	}
4803
4804
4805	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4806	/**
4807	* Check if guest state allows safe use of 32-bit switcher again.
4808	*
4809	* Segment bases and protected mode structures must be 32-bit addressable
4810	* because the 32-bit switcher will ignore high dword when writing these VMCS
4811	* fields. See @bugref{8432} for details.
4812	*
4813	* @returns true if safe, false if must continue to use the 64-bit switcher.
4814	* @param pCtx Pointer to the guest-CPU context.
4815	*
4816	* @remarks No-long-jump zone!!!
4817	*/
4818	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
4819	{
4820	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4821	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4822	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4823	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4824	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4825	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4826	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4827	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4828	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4829	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4830
4831	/* All good, bases are 32-bit. */
4832	return true;
4833	}
4834	#endif
4835
4836
4837	/**
4838	* Selects up the appropriate function to run guest code.
4839	*
4840	* @returns VBox status code.
4841	* @param pVCpu The cross context virtual CPU structure.
4842	*
4843	* @remarks No-long-jump zone!!!
4844	*/
4845	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu)
4846	{
4847	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4848	if (CPUMIsGuestInLongModeEx(pCtx))
4849	{
4850	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4851	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4852	#endif
4853	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4854	#if HC_ARCH_BITS == 32
4855	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4856	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4857	{
4858	#ifdef VBOX_STRICT
4859	if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4860	{
4861	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4862	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4863	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4864	AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4865	\| HM_CHANGED_VMX_ENTRY_CTLS
4866	\| HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4867	}
4868	#endif
4869	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4870
4871	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4872	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4873	pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4874	Log4Func(("Selected 64-bit switcher\n"));
4875	}
4876	#else
4877	/* 64-bit host. */
4878	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4879	#endif
4880	}
4881	else
4882	{
4883	/* Guest is not in long mode, use the 32-bit handler. */
4884	#if HC_ARCH_BITS == 32
4885	if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4886	&& !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4887	&& pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4888	{
4889	# ifdef VBOX_STRICT
4890	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4891	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4892	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4893	AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4894	\| HM_CHANGED_VMX_ENTRY_CTLS
4895	\| HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4896	# endif
4897	}
4898	# ifdef VBOX_ENABLE_64_BITS_GUESTS
4899	/*
4900	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4901	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4902	* switcher flag because now we know the guest is in a sane state where it's safe
4903	* to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4904	* the much faster 32-bit switcher again.
4905	*/
4906	if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4907	{
4908	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4909	Log4Func(("Selected 32-bit switcher\n"));
4910	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4911	}
4912	else
4913	{
4914	Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4915	if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4916	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
4917	{
4918	pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4919	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4920	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4921	\| HM_CHANGED_VMX_ENTRY_CTLS
4922	\| HM_CHANGED_VMX_EXIT_CTLS
4923	\| HM_CHANGED_HOST_CONTEXT);
4924	Log4Func(("Selected 32-bit switcher (safe)\n"));
4925	}
4926	}
4927	# else
4928	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4929	# endif
4930	#else
4931	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4932	#endif
4933	}
4934	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4935	return VINF_SUCCESS;
4936	}
4937
4938
4939	/**
4940	* Wrapper for running the guest code in VT-x.
4941	*
4942	* @returns VBox status code, no informational status codes.
4943	* @param pVCpu The cross context virtual CPU structure.
4944	*
4945	* @remarks No-long-jump zone!!!
4946	*/
4947	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu)
4948	{
4949	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4950	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4951	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
4952
4953	/*
4954	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4955	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4956	* callee-saved and thus the need for this XMM wrapper.
4957	*
4958	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4959	*/
4960	bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4961	/** @todo Add stats for resume vs launch. */
4962	PVM pVM = pVCpu->CTX_SUFF(pVM);
4963	#ifdef VBOX_WITH_KERNEL_USING_XMM
4964	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4965	#else
4966	int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4967	#endif
4968	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4969	return rc;
4970	}
4971
4972
4973	/**
4974	* Reports world-switch error and dumps some useful debug info.
4975	*
4976	* @param pVCpu The cross context virtual CPU structure.
4977	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4978	* @param pVmxTransient Pointer to the VMX transient structure (only
4979	* exitReason updated).
4980	*/
4981	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
4982	{
4983	Assert(pVCpu);
4984	Assert(pVmxTransient);
4985	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
4986
4987	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
4988	switch (rcVMRun)
4989	{
4990	case VERR_VMX_INVALID_VMXON_PTR:
4991	AssertFailed();
4992	break;
4993	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4994	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4995	{
4996	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4997	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4998	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
4999	AssertRC(rc);
5000
5001	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5002	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5003	Cannot do it here as we may have been long preempted. */
5004
5005	#ifdef VBOX_STRICT
5006	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5007	pVmxTransient->uExitReason));
5008	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
5009	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5010	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5011	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5012	else
5013	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5014	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5015	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5016
5017	/* VMX control bits. */
5018	uint32_t u32Val;
5019	uint64_t u64Val;
5020	RTHCUINTREG uHCReg;
5021	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5022	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5023	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5024	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5025	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
5026	{
5027	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5028	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5029	}
5030	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5031	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5032	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5033	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5034	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5035	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5036	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5037	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5038	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5039	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5040	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5041	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5042	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5043	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5044	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5045	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5046	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5047	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5048	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5049	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5050	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5051	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5052	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5053	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5054	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5055	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5056	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5057	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5058	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5059	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5060	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5061	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5062	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5063	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5064	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5065	{
5066	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5067	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5068	}
5069
5070	/* Guest bits. */
5071	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5072	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
5073	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5074	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
5075	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5076	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
5077	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5078	{
5079	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5080	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5081	}
5082
5083	/* Host bits. */
5084	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5085	Log4(("Host CR0 %#RHr\n", uHCReg));
5086	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5087	Log4(("Host CR3 %#RHr\n", uHCReg));
5088	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5089	Log4(("Host CR4 %#RHr\n", uHCReg));
5090
5091	RTGDTR HostGdtr;
5092	PCX86DESCHC pDesc;
5093	ASMGetGDTR(&HostGdtr);
5094	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5095	Log4(("Host CS %#08x\n", u32Val));
5096	if (u32Val < HostGdtr.cbGdt)
5097	{
5098	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5099	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5100	}
5101
5102	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5103	Log4(("Host DS %#08x\n", u32Val));
5104	if (u32Val < HostGdtr.cbGdt)
5105	{
5106	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5107	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5108	}
5109
5110	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5111	Log4(("Host ES %#08x\n", u32Val));
5112	if (u32Val < HostGdtr.cbGdt)
5113	{
5114	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5115	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5116	}
5117
5118	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5119	Log4(("Host FS %#08x\n", u32Val));
5120	if (u32Val < HostGdtr.cbGdt)
5121	{
5122	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5123	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5124	}
5125
5126	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5127	Log4(("Host GS %#08x\n", u32Val));
5128	if (u32Val < HostGdtr.cbGdt)
5129	{
5130	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5131	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5132	}
5133
5134	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5135	Log4(("Host SS %#08x\n", u32Val));
5136	if (u32Val < HostGdtr.cbGdt)
5137	{
5138	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5139	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5140	}
5141
5142	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5143	Log4(("Host TR %#08x\n", u32Val));
5144	if (u32Val < HostGdtr.cbGdt)
5145	{
5146	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5147	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5148	}
5149
5150	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5151	Log4(("Host TR Base %#RHv\n", uHCReg));
5152	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5153	Log4(("Host GDTR Base %#RHv\n", uHCReg));
5154	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5155	Log4(("Host IDTR Base %#RHv\n", uHCReg));
5156	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5157	Log4(("Host SYSENTER CS %#08x\n", u32Val));
5158	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5159	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5160	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5161	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5162	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5163	Log4(("Host RSP %#RHv\n", uHCReg));
5164	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5165	Log4(("Host RIP %#RHv\n", uHCReg));
5166	# if HC_ARCH_BITS == 64
5167	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5168	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5169	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5170	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5171	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5172	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5173	# endif
5174	#endif /* VBOX_STRICT */
5175	break;
5176	}
5177
5178	default:
5179	/* Impossible */
5180	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5181	break;
5182	}
5183	}
5184
5185
5186	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5187	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5188	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5189	#endif
5190	#ifdef VBOX_STRICT
5191	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5192	{
5193	switch (idxField)
5194	{
5195	case VMX_VMCS_GUEST_RIP:
5196	case VMX_VMCS_GUEST_RSP:
5197	case VMX_VMCS_GUEST_SYSENTER_EIP:
5198	case VMX_VMCS_GUEST_SYSENTER_ESP:
5199	case VMX_VMCS_GUEST_GDTR_BASE:
5200	case VMX_VMCS_GUEST_IDTR_BASE:
5201	case VMX_VMCS_GUEST_CS_BASE:
5202	case VMX_VMCS_GUEST_DS_BASE:
5203	case VMX_VMCS_GUEST_ES_BASE:
5204	case VMX_VMCS_GUEST_FS_BASE:
5205	case VMX_VMCS_GUEST_GS_BASE:
5206	case VMX_VMCS_GUEST_SS_BASE:
5207	case VMX_VMCS_GUEST_LDTR_BASE:
5208	case VMX_VMCS_GUEST_TR_BASE:
5209	case VMX_VMCS_GUEST_CR3:
5210	return true;
5211	}
5212	return false;
5213	}
5214
5215	static bool hmR0VmxIsValidReadField(uint32_t idxField)
5216	{
5217	switch (idxField)
5218	{
5219	/* Read-only fields. */
5220	case VMX_VMCS_RO_EXIT_QUALIFICATION:
5221	return true;
5222	}
5223	/* Remaining readable fields should also be writable. */
5224	return hmR0VmxIsValidWriteField(idxField);
5225	}
5226	#endif /* VBOX_STRICT */
5227
5228
5229	/**
5230	* Executes the specified handler in 64-bit mode.
5231	*
5232	* @returns VBox status code (no informational status codes).
5233	* @param pVCpu The cross context virtual CPU structure.
5234	* @param enmOp The operation to perform.
5235	* @param cParams Number of parameters.
5236	* @param paParam Array of 32-bit parameters.
5237	*/
5238	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5239	{
5240	PVM pVM = pVCpu->CTX_SUFF(pVM);
5241	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5242	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5243	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5244	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5245
5246	#ifdef VBOX_STRICT
5247	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5248	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5249
5250	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5251	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5252	#endif
5253
5254	/* Disable interrupts. */
5255	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5256
5257	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5258	RTCPUID idHostCpu = RTMpCpuId();
5259	CPUMR0SetLApic(pVCpu, idHostCpu);
5260	#endif
5261
5262	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5263	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5264
5265	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5266	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5267	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5268
5269	/* Leave VMX Root Mode. */
5270	VMXDisable();
5271
5272	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5273
5274	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5275	CPUMSetHyperEIP(pVCpu, enmOp);
5276	for (int i = (int)cParams - 1; i >= 0; i--)
5277	CPUMPushHyper(pVCpu, paParam[i]);
5278
5279	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5280
5281	/* Call the switcher. */
5282	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
5283	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5284
5285	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5286	/* Make sure the VMX instructions don't cause #UD faults. */
5287	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5288
5289	/* Re-enter VMX Root Mode */
5290	int rc2 = VMXEnable(HCPhysCpuPage);
5291	if (RT_FAILURE(rc2))
5292	{
5293	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5294	ASMSetFlags(fOldEFlags);
5295	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5296	return rc2;
5297	}
5298
5299	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5300	AssertRC(rc2);
5301	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5302	Assert(!(ASMGetFlags() & X86_EFL_IF));
5303	ASMSetFlags(fOldEFlags);
5304	return rc;
5305	}
5306
5307
5308	/**
5309	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5310	* supporting 64-bit guests.
5311	*
5312	* @returns VBox status code.
5313	* @param fResume Whether to VMLAUNCH or VMRESUME.
5314	* @param pCtx Pointer to the guest-CPU context.
5315	* @param pCache Pointer to the VMCS cache.
5316	* @param pVM The cross context VM structure.
5317	* @param pVCpu The cross context virtual CPU structure.
5318	*/
5319	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5320	{
5321	NOREF(fResume);
5322
5323	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5324	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5325
5326	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5327	pCache->uPos = 1;
5328	pCache->interPD = PGMGetInterPaeCR3(pVM);
5329	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5330	#endif
5331
5332	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5333	pCache->TestIn.HCPhysCpuPage = 0;
5334	pCache->TestIn.HCPhysVmcs = 0;
5335	pCache->TestIn.pCache = 0;
5336	pCache->TestOut.HCPhysVmcs = 0;
5337	pCache->TestOut.pCache = 0;
5338	pCache->TestOut.pCtx = 0;
5339	pCache->TestOut.eflags = 0;
5340	#else
5341	NOREF(pCache);
5342	#endif
5343
5344	uint32_t aParam[10];
5345	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5346	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5347	aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5348	aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5349	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5350	aParam[5] = 0;
5351	aParam[6] = VM_RC_ADDR(pVM, pVM);
5352	aParam[7] = 0;
5353	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5354	aParam[9] = 0;
5355
5356	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5357	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5358	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5359	#endif
5360	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5361
5362	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5363	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5364	Assert(pCtx->dr[4] == 10);
5365	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5366	#endif
5367
5368	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5369	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5370	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5371	pVCpu->hm.s.vmx.HCPhysVmcs));
5372	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5373	pCache->TestOut.HCPhysVmcs));
5374	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5375	pCache->TestOut.pCache));
5376	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5377	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5378	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5379	pCache->TestOut.pCtx));
5380	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5381	#endif
5382	NOREF(pCtx);
5383	return rc;
5384	}
5385
5386
5387	/**
5388	* Initialize the VMCS-Read cache.
5389	*
5390	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5391	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5392	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5393	* (those that have a 32-bit FULL & HIGH part).
5394	*
5395	* @returns VBox status code.
5396	* @param pVCpu The cross context virtual CPU structure.
5397	*/
5398	static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5399	{
5400	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5401	do { \
5402	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5403	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5404	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5405	++cReadFields; \
5406	} while (0)
5407
5408	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5409	uint32_t cReadFields = 0;
5410
5411	/*
5412	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5413	* and serve to indicate exceptions to the rules.
5414	*/
5415
5416	/* Guest-natural selector base fields. */
5417	#if 0
5418	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5419	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5420	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5421	#endif
5422	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5423	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5424	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5425	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5426	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5427	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5428	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5429	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5430	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5431	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5432	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5433	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5434	#if 0
5435	/* Unused natural width guest-state fields. */
5436	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5437	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5438	#endif
5439	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5440	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5441
5442	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5443	these 64-bit fields (using "FULL" and "HIGH" fields). */
5444	#if 0
5445	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5446	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5447	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5448	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5449	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5450	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5451	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5452	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5453	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5454	#endif
5455
5456	/* Natural width guest-state fields. */
5457	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5458	#if 0
5459	/* Currently unused field. */
5460	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5461	#endif
5462
5463	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5464	{
5465	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5466	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5467	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5468	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5469	}
5470	else
5471	{
5472	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5473	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5474	}
5475
5476	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5477	return VINF_SUCCESS;
5478	}
5479
5480
5481	/**
5482	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5483	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5484	* darwin, running 64-bit guests).
5485	*
5486	* @returns VBox status code.
5487	* @param pVCpu The cross context virtual CPU structure.
5488	* @param idxField The VMCS field encoding.
5489	* @param u64Val 16, 32 or 64-bit value.
5490	*/
5491	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5492	{
5493	int rc;
5494	switch (idxField)
5495	{
5496	/*
5497	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5498	*/
5499	/* 64-bit Control fields. */
5500	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5501	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5502	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5503	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5504	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5505	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5506	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5507	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5508	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
5509	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5510	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5511	case VMX_VMCS64_CTRL_EPTP_FULL:
5512	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5513	/* 64-bit Guest-state fields. */
5514	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5515	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5516	case VMX_VMCS64_GUEST_PAT_FULL:
5517	case VMX_VMCS64_GUEST_EFER_FULL:
5518	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5519	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5520	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5521	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5522	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5523	/* 64-bit Host-state fields. */
5524	case VMX_VMCS64_HOST_PAT_FULL:
5525	case VMX_VMCS64_HOST_EFER_FULL:
5526	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5527	{
5528	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5529	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5530	break;
5531	}
5532
5533	/*
5534	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5535	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5536	*/
5537	/* Natural-width Guest-state fields. */
5538	case VMX_VMCS_GUEST_CR3:
5539	case VMX_VMCS_GUEST_ES_BASE:
5540	case VMX_VMCS_GUEST_CS_BASE:
5541	case VMX_VMCS_GUEST_SS_BASE:
5542	case VMX_VMCS_GUEST_DS_BASE:
5543	case VMX_VMCS_GUEST_FS_BASE:
5544	case VMX_VMCS_GUEST_GS_BASE:
5545	case VMX_VMCS_GUEST_LDTR_BASE:
5546	case VMX_VMCS_GUEST_TR_BASE:
5547	case VMX_VMCS_GUEST_GDTR_BASE:
5548	case VMX_VMCS_GUEST_IDTR_BASE:
5549	case VMX_VMCS_GUEST_RSP:
5550	case VMX_VMCS_GUEST_RIP:
5551	case VMX_VMCS_GUEST_SYSENTER_ESP:
5552	case VMX_VMCS_GUEST_SYSENTER_EIP:
5553	{
5554	if (!(RT_HI_U32(u64Val)))
5555	{
5556	/* If this field is 64-bit, VT-x will zero out the top bits. */
5557	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5558	}
5559	else
5560	{
5561	/* Assert that only the 32->64 switcher case should ever come here. */
5562	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5563	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5564	}
5565	break;
5566	}
5567
5568	default:
5569	{
5570	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5571	rc = VERR_INVALID_PARAMETER;
5572	break;
5573	}
5574	}
5575	AssertRCReturn(rc, rc);
5576	return rc;
5577	}
5578
5579
5580	/**
5581	* Queue up a VMWRITE by using the VMCS write cache.
5582	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5583	*
5584	* @param pVCpu The cross context virtual CPU structure.
5585	* @param idxField The VMCS field encoding.
5586	* @param u64Val 16, 32 or 64-bit value.
5587	*/
5588	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5589	{
5590	AssertPtr(pVCpu);
5591	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5592
5593	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5594	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5595
5596	/* Make sure there are no duplicates. */
5597	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5598	{
5599	if (pCache->Write.aField[i] == idxField)
5600	{
5601	pCache->Write.aFieldVal[i] = u64Val;
5602	return VINF_SUCCESS;
5603	}
5604	}
5605
5606	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5607	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5608	pCache->Write.cValidEntries++;
5609	return VINF_SUCCESS;
5610	}
5611	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5612
5613
5614	/**
5615	* Sets up the usage of TSC-offsetting and updates the VMCS.
5616	*
5617	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5618	* VMX preemption timer.
5619	*
5620	* @returns VBox status code.
5621	* @param pVCpu The cross context virtual CPU structure.
5622	*
5623	* @remarks No-long-jump zone!!!
5624	*/
5625	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5626	{
5627	bool fOffsettedTsc;
5628	bool fParavirtTsc;
5629	PVM pVM = pVCpu->CTX_SUFF(pVM);
5630	uint64_t uTscOffset;
5631	if (pVM->hm.s.vmx.fUsePreemptTimer)
5632	{
5633	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5634
5635	/* Make sure the returned values have sane upper and lower boundaries. */
5636	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5637	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5638	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5639	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5640
5641	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5642	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5643	AssertRC(rc);
5644	}
5645	else
5646	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5647
5648	if (fParavirtTsc)
5649	{
5650	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5651	information before every VM-entry, hence disable it for performance sake. */
5652	#if 0
5653	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5654	AssertRC(rc);
5655	#endif
5656	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5657	}
5658
5659	uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
5660	if ( fOffsettedTsc
5661	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5662	{
5663	if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5664	{
5665	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
5666	AssertRC(rc);
5667	pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5668	}
5669
5670	if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
5671	{
5672	uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
5673	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5674	AssertRC(rc);
5675	pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5676	}
5677	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5678	}
5679	else
5680	{
5681	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5682	if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
5683	{
5684	uProcCtls \|= VMX_PROC_CTLS_RDTSC_EXIT;
5685	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5686	AssertRC(rc);
5687	pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5688	}
5689	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5690	}
5691	}
5692
5693
5694	/**
5695	* Gets the IEM exception flags for the specified vector and IDT vectoring /
5696	* VM-exit interruption info type.
5697	*
5698	* @returns The IEM exception flags.
5699	* @param uVector The event vector.
5700	* @param uVmxVectorType The VMX event type.
5701	*
5702	* @remarks This function currently only constructs flags required for
5703	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5704	* and CR2 aspects of an exception are not included).
5705	*/
5706	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5707	{
5708	uint32_t fIemXcptFlags;
5709	switch (uVmxVectorType)
5710	{
5711	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5712	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5713	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5714	break;
5715
5716	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5717	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5718	break;
5719
5720	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5721	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
5722	break;
5723
5724	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5725	{
5726	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5727	if (uVector == X86_XCPT_BP)
5728	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
5729	else if (uVector == X86_XCPT_OF)
5730	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
5731	else
5732	{
5733	fIemXcptFlags = 0;
5734	AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5735	}
5736	break;
5737	}
5738
5739	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5740	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5741	break;
5742
5743	default:
5744	fIemXcptFlags = 0;
5745	AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5746	break;
5747	}
5748	return fIemXcptFlags;
5749	}
5750
5751
5752	/**
5753	* Sets an event as a pending event to be injected into the guest.
5754	*
5755	* @param pVCpu The cross context virtual CPU structure.
5756	* @param u32IntInfo The VM-entry interruption-information field.
5757	* @param cbInstr The VM-entry instruction length in bytes (for software
5758	* interrupts, exceptions and privileged software
5759	* exceptions).
5760	* @param u32ErrCode The VM-entry exception error code.
5761	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5762	* page-fault.
5763	*
5764	* @remarks Statistics counter assumes this is a guest event being injected or
5765	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5766	* always incremented.
5767	*/
5768	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5769	RTGCUINTPTR GCPtrFaultAddress)
5770	{
5771	Assert(!pVCpu->hm.s.Event.fPending);
5772	pVCpu->hm.s.Event.fPending = true;
5773	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5774	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5775	pVCpu->hm.s.Event.cbInstr = cbInstr;
5776	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5777	}
5778
5779
5780	/**
5781	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5782	*
5783	* @param pVCpu The cross context virtual CPU structure.
5784	*/
5785	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
5786	{
5787	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
5788	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5789	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5790	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5791	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5792	}
5793
5794
5795	/**
5796	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
5797	*
5798	* @param pVCpu The cross context virtual CPU structure.
5799	*/
5800	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
5801	{
5802	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
5803	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5804	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
5805	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5806	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5807	}
5808
5809
5810	/**
5811	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
5812	*
5813	* @param pVCpu The cross context virtual CPU structure.
5814	*/
5815	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
5816	{
5817	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
5818	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5819	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
5820	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5821	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5822	}
5823
5824
5825	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5826	/**
5827	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
5828	*
5829	* @param pVCpu The cross context virtual CPU structure.
5830	* @param u32ErrCode The error code for the general-protection exception.
5831	*/
5832	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
5833	{
5834	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
5835	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5836	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5837	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5838	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
5839	}
5840
5841
5842	/**
5843	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
5844	*
5845	* @param pVCpu The cross context virtual CPU structure.
5846	* @param u32ErrCode The error code for the stack exception.
5847	*/
5848	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
5849	{
5850	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
5851	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5852	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5853	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5854	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
5855	}
5856
5857
5858	/**
5859	* Decodes the memory operand of an instruction that caused a VM-exit.
5860	*
5861	* The VM-exit qualification field provides the displacement field for memory
5862	* operand instructions, if any.
5863	*
5864	* @returns Strict VBox status code (i.e. informational status codes too).
5865	* @retval VINF_SUCCESS if the operand was successfully decoded.
5866	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
5867	* operand.
5868	* @param pVCpu The cross context virtual CPU structure.
5869	* @param uExitInstrInfo The VM-exit instruction information field.
5870	* @param enmMemAccess The memory operand's access type (read or write).
5871	* @param GCPtrDisp The instruction displacement field, if any. For
5872	* RIP-relative addressing pass RIP + displacement here.
5873	* @param pGCPtrMem Where to store the effective destination memory address.
5874	*/
5875	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
5876	PRTGCPTR pGCPtrMem)
5877	{
5878	Assert(pGCPtrMem);
5879	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
5880	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER \| CPUMCTX_EXTRN_CR0);
5881
5882	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
5883	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
5884	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
5885
5886	VMXEXITINSTRINFO ExitInstrInfo;
5887	ExitInstrInfo.u = uExitInstrInfo;
5888	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
5889	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
5890	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
5891	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
5892	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
5893	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
5894	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
5895	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
5896	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
5897
5898	/*
5899	* Validate instruction information.
5900	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
5901	*/
5902	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
5903	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
5904	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
5905	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
5906	AssertLogRelMsgReturn(fIsMemOperand,
5907	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
5908
5909	/*
5910	* Compute the complete effective address.
5911	*
5912	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
5913	* See AMD spec. 4.5.2 "Segment Registers".
5914	*/
5915	RTGCPTR GCPtrMem = GCPtrDisp;
5916	if (fBaseRegValid)
5917	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
5918	if (fIdxRegValid)
5919	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
5920
5921	RTGCPTR const GCPtrOff = GCPtrMem;
5922	if ( !fIsLongMode
5923	\|\| iSegReg >= X86_SREG_FS)
5924	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
5925	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
5926
5927	/*
5928	* Validate effective address.
5929	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
5930	*/
5931	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
5932	Assert(cbAccess > 0);
5933	if (fIsLongMode)
5934	{
5935	if (X86_IS_CANONICAL(GCPtrMem))
5936	{
5937	*pGCPtrMem = GCPtrMem;
5938	return VINF_SUCCESS;
5939	}
5940
5941	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
5942	* "Data Limit Checks in 64-bit Mode". */
5943	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
5944	hmR0VmxSetPendingXcptGP(pVCpu, 0);
5945	return VINF_HM_PENDING_XCPT;
5946	}
5947
5948	/*
5949	* This is a watered down version of iemMemApplySegment().
5950	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
5951	* and segment CPL/DPL checks are skipped.
5952	*/
5953	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
5954	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
5955	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
5956
5957	/* Check if the segment is present and usable. */
5958	if ( pSel->Attr.n.u1Present
5959	&& !pSel->Attr.n.u1Unusable)
5960	{
5961	Assert(pSel->Attr.n.u1DescType);
5962	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
5963	{
5964	/* Check permissions for the data segment. */
5965	if ( enmMemAccess == VMXMEMACCESS_WRITE
5966	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
5967	{
5968	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
5969	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
5970	return VINF_HM_PENDING_XCPT;
5971	}
5972
5973	/* Check limits if it's a normal data segment. */
5974	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
5975	{
5976	if ( GCPtrFirst32 > pSel->u32Limit
5977	\|\| GCPtrLast32 > pSel->u32Limit)
5978	{
5979	Log4Func(("Data segment limit exceeded."
5980	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
5981	GCPtrLast32, pSel->u32Limit));
5982	if (iSegReg == X86_SREG_SS)
5983	hmR0VmxSetPendingXcptSS(pVCpu, 0);
5984	else
5985	hmR0VmxSetPendingXcptGP(pVCpu, 0);
5986	return VINF_HM_PENDING_XCPT;
5987	}
5988	}
5989	else
5990	{
5991	/* Check limits if it's an expand-down data segment.
5992	Note! The upper boundary is defined by the B bit, not the G bit! */
5993	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
5994	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
5995	{
5996	Log4Func(("Expand-down data segment limit exceeded."
5997	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
5998	GCPtrLast32, pSel->u32Limit));
5999	if (iSegReg == X86_SREG_SS)
6000	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6001	else
6002	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6003	return VINF_HM_PENDING_XCPT;
6004	}
6005	}
6006	}
6007	else
6008	{
6009	/* Check permissions for the code segment. */
6010	if ( enmMemAccess == VMXMEMACCESS_WRITE
6011	\|\| ( enmMemAccess == VMXMEMACCESS_READ
6012	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
6013	{
6014	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6015	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6016	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6017	return VINF_HM_PENDING_XCPT;
6018	}
6019
6020	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6021	if ( GCPtrFirst32 > pSel->u32Limit
6022	\|\| GCPtrLast32 > pSel->u32Limit)
6023	{
6024	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6025	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6026	if (iSegReg == X86_SREG_SS)
6027	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6028	else
6029	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6030	return VINF_HM_PENDING_XCPT;
6031	}
6032	}
6033	}
6034	else
6035	{
6036	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6037	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6038	return VINF_HM_PENDING_XCPT;
6039	}
6040
6041	*pGCPtrMem = GCPtrMem;
6042	return VINF_SUCCESS;
6043	}
6044
6045
6046	/**
6047	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6048	* guest attempting to execute a VMX instruction.
6049	*
6050	* @returns Strict VBox status code (i.e. informational status codes too).
6051	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
6052	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
6053	*
6054	* @param pVCpu The cross context virtual CPU structure.
6055	* @param uExitReason The VM-exit reason.
6056	*
6057	* @todo NstVmx: Document other error codes when VM-exit is implemented.
6058	* @remarks No-long-jump zone!!!
6059	*/
6060	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
6061	{
6062	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
6063	\| CPUMCTX_EXTRN_HWVIRT);
6064
6065	if ( CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx)
6066	\|\| ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6067	&& !CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)))
6068	{
6069	Log4Func(("In real/v86-mode or long-mode outside 64-bit code segment -> #UD\n"));
6070	hmR0VmxSetPendingXcptUD(pVCpu);
6071	return VINF_HM_PENDING_XCPT;
6072	}
6073
6074	if (uExitReason == VMX_EXIT_VMXON)
6075	{
6076	/*
6077	* We check CR4.VMXE because it is required to be always set while in VMX operation
6078	* by physical CPUs and our CR4 read shadow is only consulted when executing specific
6079	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
6080	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
6081	*/
6082	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
6083	{
6084	Log4Func(("CR4.VMXE is not set -> #UD\n"));
6085	hmR0VmxSetPendingXcptUD(pVCpu);
6086	return VINF_HM_PENDING_XCPT;
6087	}
6088	}
6089	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
6090	{
6091	/*
6092	* The guest has not entered VMX operation but attempted to execute a VMX instruction
6093	* (other than VMXON), we need to raise a #UD.
6094	*/
6095	Log4Func(("Not in VMX root mode -> #UD\n"));
6096	hmR0VmxSetPendingXcptUD(pVCpu);
6097	return VINF_HM_PENDING_XCPT;
6098	}
6099
6100	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
6101	{
6102	/*
6103	* The nested-guest attempted to execute a VMX instruction, cause a VM-exit and let
6104	* the guest hypervisor deal with it.
6105	*/
6106	/** @todo NSTVMX: Trigger a VM-exit */
6107	}
6108
6109	/*
6110	* VMX instructions require CPL 0 except in VMX non-root mode where the VM-exit intercept
6111	* (above) takes preceedence over the CPL check.
6112	*/
6113	if (CPUMGetGuestCPL(pVCpu) > 0)
6114	{
6115	Log4Func(("CPL > 0 -> #GP(0)\n"));
6116	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6117	return VINF_HM_PENDING_XCPT;
6118	}
6119
6120	return VINF_SUCCESS;
6121	}
6122
6123	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
6124
6125
6126	/**
6127	* Handle a condition that occurred while delivering an event through the guest
6128	* IDT.
6129	*
6130	* @returns Strict VBox status code (i.e. informational status codes too).
6131	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
6132	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
6133	* to continue execution of the guest which will delivery the \#DF.
6134	* @retval VINF_EM_RESET if we detected a triple-fault condition.
6135	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
6136	*
6137	* @param pVCpu The cross context virtual CPU structure.
6138	* @param pVmxTransient Pointer to the VMX transient structure.
6139	*
6140	* @remarks No-long-jump zone!!!
6141	*/
6142	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6143	{
6144	uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
6145
6146	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
6147	rc2 \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
6148	AssertRCReturn(rc2, rc2);
6149
6150	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
6151	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6152	{
6153	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
6154	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
6155
6156	/*
6157	* If the event was a software interrupt (generated with INT n) or a software exception
6158	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
6159	* can handle the VM-exit and continue guest execution which will re-execute the
6160	* instruction rather than re-injecting the exception, as that can cause premature
6161	* trips to ring-3 before injection and involve TRPM which currently has no way of
6162	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
6163	* the problem).
6164	*/
6165	IEMXCPTRAISE enmRaise;
6166	IEMXCPTRAISEINFO fRaiseInfo;
6167	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6168	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6169	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6170	{
6171	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
6172	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6173	}
6174	else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
6175	{
6176	uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
6177	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
6178	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
6179	/** @todo Make AssertMsgReturn as just AssertMsg later. */
6180	AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
6181	("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
6182	uExitVectorType), VERR_VMX_IPE_5);
6183
6184	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
6185
6186	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
6187	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
6188	{
6189	pVmxTransient->fVectoringPF = true;
6190	enmRaise = IEMXCPTRAISE_PREV_EVENT;
6191	}
6192	}
6193	else
6194	{
6195	/*
6196	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
6197	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
6198	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
6199	*/
6200	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6201	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6202	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
6203	enmRaise = IEMXCPTRAISE_PREV_EVENT;
6204	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6205	}
6206
6207	/*
6208	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
6209	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
6210	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
6211	* subsequent VM-entry would fail.
6212	*
6213	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6214	*/
6215	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
6216	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6217	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
6218	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
6219	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
6220	{
6221	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6222	}
6223
6224	switch (enmRaise)
6225	{
6226	case IEMXCPTRAISE_CURRENT_XCPT:
6227	{
6228	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
6229	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
6230	Assert(rcStrict == VINF_SUCCESS);
6231	break;
6232	}
6233
6234	case IEMXCPTRAISE_PREV_EVENT:
6235	{
6236	uint32_t u32ErrCode;
6237	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
6238	{
6239	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6240	AssertRCReturn(rc2, rc2);
6241	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6242	}
6243	else
6244	u32ErrCode = 0;
6245
6246	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
6247	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6248	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6249	0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
6250
6251	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
6252	pVCpu->hm.s.Event.u32ErrCode));
6253	Assert(rcStrict == VINF_SUCCESS);
6254	break;
6255	}
6256
6257	case IEMXCPTRAISE_REEXEC_INSTR:
6258	Assert(rcStrict == VINF_SUCCESS);
6259	break;
6260
6261	case IEMXCPTRAISE_DOUBLE_FAULT:
6262	{
6263	/*
6264	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
6265	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
6266	*/
6267	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6268	{
6269	pVmxTransient->fVectoringDoublePF = true;
6270	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
6271	pVCpu->cpum.GstCtx.cr2));
6272	rcStrict = VINF_SUCCESS;
6273	}
6274	else
6275	{
6276	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6277	hmR0VmxSetPendingXcptDF(pVCpu);
6278	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
6279	uIdtVector, uExitVector));
6280	rcStrict = VINF_HM_DOUBLE_FAULT;
6281	}
6282	break;
6283	}
6284
6285	case IEMXCPTRAISE_TRIPLE_FAULT:
6286	{
6287	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
6288	rcStrict = VINF_EM_RESET;
6289	break;
6290	}
6291
6292	case IEMXCPTRAISE_CPU_HANG:
6293	{
6294	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
6295	rcStrict = VERR_EM_GUEST_CPU_HANG;
6296	break;
6297	}
6298
6299	default:
6300	{
6301	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6302	rcStrict = VERR_VMX_IPE_2;
6303	break;
6304	}
6305	}
6306	}
6307	else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6308	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6309	&& uExitVector != X86_XCPT_DF
6310	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
6311	{
6312	/*
6313	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6314	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6315	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6316	*/
6317	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6318	{
6319	Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
6320	VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6321	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6322	}
6323	}
6324
6325	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
6326	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
6327	return rcStrict;
6328	}
6329
6330
6331	/**
6332	* Imports a guest segment register from the current VMCS into
6333	* the guest-CPU context.
6334	*
6335	* @returns VBox status code.
6336	* @param pVCpu The cross context virtual CPU structure.
6337	* @param idxSel Index of the selector in the VMCS.
6338	* @param idxLimit Index of the segment limit in the VMCS.
6339	* @param idxBase Index of the segment base in the VMCS.
6340	* @param idxAccess Index of the access rights of the segment in the VMCS.
6341	* @param pSelReg Pointer to the segment selector.
6342	*
6343	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
6344	* do not log!
6345	*
6346	* @remarks Never call this function directly!!! Use the
6347	* HMVMX_IMPORT_SREG() macro as that takes care
6348	* of whether to read from the VMCS cache or not.
6349	*/
6350	static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6351	PCPUMSELREG pSelReg)
6352	{
6353	NOREF(pVCpu);
6354
6355	uint32_t u32Sel;
6356	uint32_t u32Limit;
6357	uint32_t u32Attr;
6358	uint64_t u64Base;
6359	int rc = VMXReadVmcs32(idxSel, &u32Sel);
6360	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
6361	rc \|= VMXReadVmcs32(idxAccess, &u32Attr);
6362	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6363	AssertRCReturn(rc, rc);
6364
6365	pSelReg->Sel = (uint16_t)u32Sel;
6366	pSelReg->ValidSel = (uint16_t)u32Sel;
6367	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6368	pSelReg->u32Limit = u32Limit;
6369	pSelReg->u64Base = u64Base;
6370	pSelReg->Attr.u = u32Attr;
6371
6372	/*
6373	* If VT-x marks the segment as unusable, most other bits remain undefined:
6374	* - For CS the L, D and G bits have meaning.
6375	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6376	* - For the remaining data segments no bits are defined.
6377	*
6378	* The present bit and the unusable bit has been observed to be set at the
6379	* same time (the selector was supposed to be invalid as we started executing
6380	* a V8086 interrupt in ring-0).
6381	*
6382	* What should be important for the rest of the VBox code, is that the P bit is
6383	* cleared. Some of the other VBox code recognizes the unusable bit, but
6384	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6385	* safe side here, we'll strip off P and other bits we don't care about. If
6386	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6387	*
6388	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6389	*/
6390	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6391	{
6392	Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6393
6394	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6395	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6396	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6397	#ifdef VBOX_STRICT
6398	VMMRZCallRing3Disable(pVCpu);
6399	Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6400	# ifdef DEBUG_bird
6401	AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6402	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6403	idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6404	# endif
6405	VMMRZCallRing3Enable(pVCpu);
6406	#endif
6407	}
6408	return VINF_SUCCESS;
6409	}
6410
6411
6412	/**
6413	* Imports the guest RIP from the VMCS back into the guest-CPU context.
6414	*
6415	* @returns VBox status code.
6416	* @param pVCpu The cross context virtual CPU structure.
6417	*
6418	* @remarks Called with interrupts and/or preemption disabled, should not assert!
6419	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6420	* instead!!!
6421	*/
6422	DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6423	{
6424	uint64_t u64Val;
6425	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6426	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6427	{
6428	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6429	if (RT_SUCCESS(rc))
6430	{
6431	pCtx->rip = u64Val;
6432	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
6433	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6434	}
6435	return rc;
6436	}
6437	return VINF_SUCCESS;
6438	}
6439
6440
6441	/**
6442	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6443	*
6444	* @returns VBox status code.
6445	* @param pVCpu The cross context virtual CPU structure.
6446	*
6447	* @remarks Called with interrupts and/or preemption disabled, should not assert!
6448	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6449	* instead!!!
6450	*/
6451	DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6452	{
6453	uint32_t u32Val;
6454	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6455	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6456	{
6457	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6458	if (RT_SUCCESS(rc))
6459	{
6460	pCtx->eflags.u32 = u32Val;
6461
6462	/* Restore eflags for real-on-v86-mode hack. */
6463	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6464	{
6465	pCtx->eflags.Bits.u1VM = 0;
6466	pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6467	}
6468	}
6469	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6470	return rc;
6471	}
6472	return VINF_SUCCESS;
6473	}
6474
6475
6476	/**
6477	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6478	* context.
6479	*
6480	* @returns VBox status code.
6481	* @param pVCpu The cross context virtual CPU structure.
6482	*
6483	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
6484	* do not log!
6485	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6486	* instead!!!
6487	*/
6488	DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6489	{
6490	uint32_t u32Val;
6491	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6492	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
6493	if (RT_SUCCESS(rc))
6494	{
6495	/*
6496	* We additionally have a requirement to import RIP, RFLAGS depending on whether we
6497	* might need them in hmR0VmxEvaluatePendingEvent().
6498	*/
6499	if (!u32Val)
6500	{
6501	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6502	{
6503	rc = hmR0VmxImportGuestRip(pVCpu);
6504	rc \|= hmR0VmxImportGuestRFlags(pVCpu);
6505	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6506	}
6507
6508	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6509	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6510	}
6511	else
6512	{
6513	rc = hmR0VmxImportGuestRip(pVCpu);
6514	rc \|= hmR0VmxImportGuestRFlags(pVCpu);
6515
6516	if (u32Val & ( VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
6517	\| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
6518	{
6519	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6520	}
6521	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6522	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6523
6524	if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
6525	{
6526	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6527	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6528	}
6529	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6530	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6531	}
6532	}
6533	return rc;
6534	}
6535
6536
6537	/**
6538	* Worker for VMXR0ImportStateOnDemand.
6539	*
6540	* @returns VBox status code.
6541	* @param pVCpu The cross context virtual CPU structure.
6542	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6543	*/
6544	static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6545	{
6546	#define VMXLOCAL_BREAK_RC(a_rc) \
6547	if (RT_FAILURE(a_rc)) \
6548	break
6549
6550	int rc = VINF_SUCCESS;
6551	PVM pVM = pVCpu->CTX_SUFF(pVM);
6552	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6553	uint64_t u64Val;
6554	uint32_t u32Val;
6555
6556	Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6557	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6558
6559	/*
6560	* We disable interrupts to make the updating of the state and in particular
6561	* the fExtrn modification atomic wrt to preemption hooks.
6562	*/
6563	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6564
6565	fWhat &= pCtx->fExtrn;
6566	if (fWhat)
6567	{
6568	do
6569	{
6570	if (fWhat & CPUMCTX_EXTRN_RIP)
6571	{
6572	rc = hmR0VmxImportGuestRip(pVCpu);
6573	VMXLOCAL_BREAK_RC(rc);
6574	}
6575
6576	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6577	{
6578	rc = hmR0VmxImportGuestRFlags(pVCpu);
6579	VMXLOCAL_BREAK_RC(rc);
6580	}
6581
6582	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6583	{
6584	rc = hmR0VmxImportGuestIntrState(pVCpu);
6585	VMXLOCAL_BREAK_RC(rc);
6586	}
6587
6588	if (fWhat & CPUMCTX_EXTRN_RSP)
6589	{
6590	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6591	VMXLOCAL_BREAK_RC(rc);
6592	pCtx->rsp = u64Val;
6593	}
6594
6595	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6596	{
6597	if (fWhat & CPUMCTX_EXTRN_CS)
6598	{
6599	rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6600	rc \|= hmR0VmxImportGuestRip(pVCpu);
6601	VMXLOCAL_BREAK_RC(rc);
6602	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6603	pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6604	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true);
6605	}
6606	if (fWhat & CPUMCTX_EXTRN_SS)
6607	{
6608	rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6609	VMXLOCAL_BREAK_RC(rc);
6610	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6611	pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6612	}
6613	if (fWhat & CPUMCTX_EXTRN_DS)
6614	{
6615	rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6616	VMXLOCAL_BREAK_RC(rc);
6617	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6618	pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6619	}
6620	if (fWhat & CPUMCTX_EXTRN_ES)
6621	{
6622	rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6623	VMXLOCAL_BREAK_RC(rc);
6624	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6625	pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6626	}
6627	if (fWhat & CPUMCTX_EXTRN_FS)
6628	{
6629	rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6630	VMXLOCAL_BREAK_RC(rc);
6631	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6632	pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6633	}
6634	if (fWhat & CPUMCTX_EXTRN_GS)
6635	{
6636	rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6637	VMXLOCAL_BREAK_RC(rc);
6638	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6639	pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6640	}
6641	}
6642
6643	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6644	{
6645	if (fWhat & CPUMCTX_EXTRN_LDTR)
6646	{
6647	rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6648	VMXLOCAL_BREAK_RC(rc);
6649	}
6650
6651	if (fWhat & CPUMCTX_EXTRN_GDTR)
6652	{
6653	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6654	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6655	VMXLOCAL_BREAK_RC(rc);
6656	pCtx->gdtr.pGdt = u64Val;
6657	pCtx->gdtr.cbGdt = u32Val;
6658	}
6659
6660	/* Guest IDTR. */
6661	if (fWhat & CPUMCTX_EXTRN_IDTR)
6662	{
6663	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6664	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6665	VMXLOCAL_BREAK_RC(rc);
6666	pCtx->idtr.pIdt = u64Val;
6667	pCtx->idtr.cbIdt = u32Val;
6668	}
6669
6670	/* Guest TR. */
6671	if (fWhat & CPUMCTX_EXTRN_TR)
6672	{
6673	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6674	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6675	{
6676	rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6677	VMXLOCAL_BREAK_RC(rc);
6678	}
6679	}
6680	}
6681
6682	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6683	{
6684	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6685	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6686	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6687	pCtx->SysEnter.cs = u32Val;
6688	VMXLOCAL_BREAK_RC(rc);
6689	}
6690
6691	#if HC_ARCH_BITS == 64
6692	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6693	{
6694	if ( pVM->hm.s.fAllow64BitGuests
6695	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6696	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6697	}
6698
6699	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6700	{
6701	if ( pVM->hm.s.fAllow64BitGuests
6702	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6703	{
6704	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6705	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6706	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6707	}
6708	}
6709	#endif
6710
6711	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
6712	#if HC_ARCH_BITS == 32
6713	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
6714	#endif
6715	)
6716	{
6717	PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6718	uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6719	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6720	{
6721	switch (pMsr->u32Msr)
6722	{
6723	#if HC_ARCH_BITS == 32
6724	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6725	case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6726	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6727	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6728	#endif
6729	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6730	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6731	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6732	default:
6733	{
6734	pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6735	ASMSetFlags(fEFlags);
6736	AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6737	cMsrs));
6738	return VERR_HM_UNEXPECTED_LD_ST_MSR;
6739	}
6740	}
6741	}
6742	}
6743
6744	if (fWhat & CPUMCTX_EXTRN_DR7)
6745	{
6746	if (!pVCpu->hm.s.fUsingHyperDR7)
6747	{
6748	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6749	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6750	VMXLOCAL_BREAK_RC(rc);
6751	pCtx->dr[7] = u32Val;
6752	}
6753	}
6754
6755	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6756	{
6757	uint32_t u32Shadow;
6758	if (fWhat & CPUMCTX_EXTRN_CR0)
6759	{
6760	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6761	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6762	VMXLOCAL_BREAK_RC(rc);
6763	u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr0Mask)
6764	\| (u32Shadow & pVCpu->hm.s.vmx.u32Cr0Mask);
6765	VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6766	CPUMSetGuestCR0(pVCpu, u32Val);
6767	VMMRZCallRing3Enable(pVCpu);
6768	}
6769
6770	if (fWhat & CPUMCTX_EXTRN_CR4)
6771	{
6772	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6773	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6774	VMXLOCAL_BREAK_RC(rc);
6775	u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr4Mask)
6776	\| (u32Shadow & pVCpu->hm.s.vmx.u32Cr4Mask);
6777	CPUMSetGuestCR4(pVCpu, u32Val);
6778	}
6779
6780	if (fWhat & CPUMCTX_EXTRN_CR3)
6781	{
6782	/* CR0.PG bit changes are always intercepted, so it's up to date. */
6783	if ( pVM->hm.s.vmx.fUnrestrictedGuest
6784	\|\| ( pVM->hm.s.fNestedPaging
6785	&& CPUMIsGuestPagingEnabledEx(pCtx)))
6786	{
6787	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6788	if (pCtx->cr3 != u64Val)
6789	{
6790	CPUMSetGuestCR3(pVCpu, u64Val);
6791	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6792	}
6793
6794	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6795	Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6796	if (CPUMIsGuestInPAEModeEx(pCtx))
6797	{
6798	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6799	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6800	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6801	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6802	VMXLOCAL_BREAK_RC(rc);
6803	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6804	}
6805	}
6806	}
6807	}
6808	} while (0);
6809
6810	if (RT_SUCCESS(rc))
6811	{
6812	/* Update fExtrn. */
6813	pCtx->fExtrn &= ~fWhat;
6814
6815	/* If everything has been imported, clear the HM keeper bit. */
6816	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6817	{
6818	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6819	Assert(!pCtx->fExtrn);
6820	}
6821	}
6822	}
6823	else
6824	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6825
6826	ASMSetFlags(fEFlags);
6827
6828	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6829
6830	/*
6831	* Honor any pending CR3 updates.
6832	*
6833	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6834	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6835	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6836	*
6837	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6838	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6839	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6840	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6841	*
6842	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6843	*/
6844	if (VMMRZCallRing3IsEnabled(pVCpu))
6845	{
6846	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6847	{
6848	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6849	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6850	}
6851
6852	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6853	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6854
6855	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6856	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6857	}
6858
6859	return VINF_SUCCESS;
6860	#undef VMXLOCAL_BREAK_RC
6861	}
6862
6863
6864	/**
6865	* Saves the guest state from the VMCS into the guest-CPU context.
6866	*
6867	* @returns VBox status code.
6868	* @param pVCpu The cross context virtual CPU structure.
6869	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6870	*/
6871	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6872	{
6873	return hmR0VmxImportGuestState(pVCpu, fWhat);
6874	}
6875
6876
6877	/**
6878	* Check per-VM and per-VCPU force flag actions that require us to go back to
6879	* ring-3 for one reason or another.
6880	*
6881	* @returns Strict VBox status code (i.e. informational status codes too)
6882	* @retval VINF_SUCCESS if we don't have any actions that require going back to
6883	* ring-3.
6884	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6885	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6886	* interrupts)
6887	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6888	* all EMTs to be in ring-3.
6889	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6890	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6891	* to the EM loop.
6892	*
6893	* @param pVCpu The cross context virtual CPU structure.
6894	* @param fStepping Running in hmR0VmxRunGuestCodeStep().
6895	*/
6896	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
6897	{
6898	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6899
6900	/*
6901	* Anything pending? Should be more likely than not if we're doing a good job.
6902	*/
6903	PVM pVM = pVCpu->CTX_SUFF(pVM);
6904	if ( !fStepping
6905	? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6906	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6907	: !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6908	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6909	return VINF_SUCCESS;
6910
6911	/* Pending PGM C3 sync. */
6912	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6913	{
6914	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6915	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
6916	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
6917	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6918	if (rcStrict2 != VINF_SUCCESS)
6919	{
6920	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6921	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6922	return rcStrict2;
6923	}
6924	}
6925
6926	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6927	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6928	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6929	{
6930	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6931	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6932	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6933	return rc2;
6934	}
6935
6936	/* Pending VM request packets, such as hardware interrupts. */
6937	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6938	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6939	{
6940	Log4Func(("Pending VM request forcing us back to ring-3\n"));
6941	return VINF_EM_PENDING_REQUEST;
6942	}
6943
6944	/* Pending PGM pool flushes. */
6945	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6946	{
6947	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6948	return VINF_PGM_POOL_FLUSH_PENDING;
6949	}
6950
6951	/* Pending DMA requests. */
6952	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6953	{
6954	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6955	return VINF_EM_RAW_TO_R3;
6956	}
6957
6958	return VINF_SUCCESS;
6959	}
6960
6961
6962	/**
6963	* Converts any TRPM trap into a pending HM event. This is typically used when
6964	* entering from ring-3 (not longjmp returns).
6965	*
6966	* @param pVCpu The cross context virtual CPU structure.
6967	*/
6968	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6969	{
6970	Assert(TRPMHasTrap(pVCpu));
6971	Assert(!pVCpu->hm.s.Event.fPending);
6972
6973	uint8_t uVector;
6974	TRPMEVENT enmTrpmEvent;
6975	RTGCUINT uErrCode;
6976	RTGCUINTPTR GCPtrFaultAddress;
6977	uint8_t cbInstr;
6978
6979	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6980	AssertRC(rc);
6981
6982	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6983	uint32_t u32IntInfo = uVector \| VMX_EXIT_INT_INFO_VALID;
6984	if (enmTrpmEvent == TRPM_TRAP)
6985	{
6986	switch (uVector)
6987	{
6988	case X86_XCPT_NMI:
6989	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6990	break;
6991
6992	case X86_XCPT_BP:
6993	case X86_XCPT_OF:
6994	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6995	break;
6996
6997	case X86_XCPT_PF:
6998	case X86_XCPT_DF:
6999	case X86_XCPT_TS:
7000	case X86_XCPT_NP:
7001	case X86_XCPT_SS:
7002	case X86_XCPT_GP:
7003	case X86_XCPT_AC:
7004	u32IntInfo \|= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7005	RT_FALL_THRU();
7006	default:
7007	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7008	break;
7009	}
7010	}
7011	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7012	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7013	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7014	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7015	else
7016	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7017
7018	rc = TRPMResetTrap(pVCpu);
7019	AssertRC(rc);
7020	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7021	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7022
7023	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7024	}
7025
7026
7027	/**
7028	* Converts the pending HM event into a TRPM trap.
7029	*
7030	* @param pVCpu The cross context virtual CPU structure.
7031	*/
7032	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7033	{
7034	Assert(pVCpu->hm.s.Event.fPending);
7035
7036	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7037	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7038	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
7039	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7040
7041	/* If a trap was already pending, we did something wrong! */
7042	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7043
7044	TRPMEVENT enmTrapType;
7045	switch (uVectorType)
7046	{
7047	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7048	enmTrapType = TRPM_HARDWARE_INT;
7049	break;
7050
7051	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7052	enmTrapType = TRPM_SOFTWARE_INT;
7053	break;
7054
7055	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7056	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7057	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7058	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7059	enmTrapType = TRPM_TRAP;
7060	break;
7061
7062	default:
7063	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7064	enmTrapType = TRPM_32BIT_HACK;
7065	break;
7066	}
7067
7068	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7069
7070	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7071	AssertRC(rc);
7072
7073	if (fErrorCodeValid)
7074	TRPMSetErrorCode(pVCpu, uErrorCode);
7075
7076	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7077	&& uVector == X86_XCPT_PF)
7078	{
7079	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7080	}
7081	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7082	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7083	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7084	{
7085	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7086	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
7087	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7088	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7089	}
7090
7091	/* Clear any pending events from the VMCS. */
7092	VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7093	VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, 0);
7094
7095	/* We're now done converting the pending event. */
7096	pVCpu->hm.s.Event.fPending = false;
7097	}
7098
7099
7100	/**
7101	* Does the necessary state syncing before returning to ring-3 for any reason
7102	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7103	*
7104	* @returns VBox status code.
7105	* @param pVCpu The cross context virtual CPU structure.
7106	* @param fImportState Whether to import the guest state from the VMCS back
7107	* to the guest-CPU context.
7108	*
7109	* @remarks No-long-jmp zone!!!
7110	*/
7111	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
7112	{
7113	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7114	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7115
7116	RTCPUID idCpu = RTMpCpuId();
7117	Log4Func(("HostCpuId=%u\n", idCpu));
7118
7119	/*
7120	* !!! IMPORTANT !!!
7121	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7122	*/
7123
7124	/* Save the guest state if necessary. */
7125	if (fImportState)
7126	{
7127	int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7128	AssertRCReturn(rc, rc);
7129	}
7130
7131	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
7132	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7133	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
7134
7135	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
7136	#ifdef VBOX_STRICT
7137	if (CPUMIsHyperDebugStateActive(pVCpu))
7138	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
7139	#endif
7140	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7141	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7142	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7143
7144	#if HC_ARCH_BITS == 64
7145	/* Restore host-state bits that VT-x only restores partially. */
7146	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7147	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7148	{
7149	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7150	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7151	}
7152	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7153	#endif
7154
7155	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7156	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7157	{
7158	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
7159	if (!fImportState)
7160	{
7161	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
7162	AssertRCReturn(rc, rc);
7163	}
7164	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7165	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7166	}
7167	else
7168	pVCpu->hm.s.vmx.fLazyMsrs = 0;
7169
7170	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7171	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7172
7173	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7174	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
7175	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
7176	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
7177	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
7178	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7179	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7180	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7181	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7182
7183	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7184
7185	/** @todo This partially defeats the purpose of having preemption hooks.
7186	* The problem is, deregistering the hooks should be moved to a place that
7187	* lasts until the EMT is about to be destroyed not everytime while leaving HM
7188	* context.
7189	*/
7190	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7191	{
7192	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7193	AssertRCReturn(rc, rc);
7194
7195	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7196	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7197	}
7198	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7199	NOREF(idCpu);
7200
7201	return VINF_SUCCESS;
7202	}
7203
7204
7205	/**
7206	* Leaves the VT-x session.
7207	*
7208	* @returns VBox status code.
7209	* @param pVCpu The cross context virtual CPU structure.
7210	*
7211	* @remarks No-long-jmp zone!!!
7212	*/
7213	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
7214	{
7215	HM_DISABLE_PREEMPT(pVCpu);
7216	HMVMX_ASSERT_CPU_SAFE(pVCpu);
7217	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7218	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7219
7220	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7221	and done this from the VMXR0ThreadCtxCallback(). */
7222	if (!pVCpu->hm.s.fLeaveDone)
7223	{
7224	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
7225	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7226	pVCpu->hm.s.fLeaveDone = true;
7227	}
7228	Assert(!pVCpu->cpum.GstCtx.fExtrn);
7229
7230	/*
7231	* !!! IMPORTANT !!!
7232	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7233	*/
7234
7235	/* Deregister hook now that we've left HM context before re-enabling preemption. */
7236	/** @todo Deregistering here means we need to VMCLEAR always
7237	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
7238	* for calling VMMR0ThreadCtxHookDisable here! */
7239	VMMR0ThreadCtxHookDisable(pVCpu);
7240
7241	/* Leave HM context. This takes care of local init (term). */
7242	int rc = HMR0LeaveCpu(pVCpu);
7243
7244	HM_RESTORE_PREEMPT();
7245	return rc;
7246	}
7247
7248
7249	/**
7250	* Does the necessary state syncing before doing a longjmp to ring-3.
7251	*
7252	* @returns VBox status code.
7253	* @param pVCpu The cross context virtual CPU structure.
7254	*
7255	* @remarks No-long-jmp zone!!!
7256	*/
7257	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
7258	{
7259	return hmR0VmxLeaveSession(pVCpu);
7260	}
7261
7262
7263	/**
7264	* Take necessary actions before going back to ring-3.
7265	*
7266	* An action requires us to go back to ring-3. This function does the necessary
7267	* steps before we can safely return to ring-3. This is not the same as longjmps
7268	* to ring-3, this is voluntary and prepares the guest so it may continue
7269	* executing outside HM (recompiler/IEM).
7270	*
7271	* @returns VBox status code.
7272	* @param pVCpu The cross context virtual CPU structure.
7273	* @param rcExit The reason for exiting to ring-3. Can be
7274	* VINF_VMM_UNKNOWN_RING3_CALL.
7275	*/
7276	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
7277	{
7278	Assert(pVCpu);
7279	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7280
7281	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7282	{
7283	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7284	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
7285	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7286	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7287	}
7288
7289	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7290	VMMRZCallRing3Disable(pVCpu);
7291	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
7292
7293	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7294	if (pVCpu->hm.s.Event.fPending)
7295	{
7296	hmR0VmxPendingEventToTrpmTrap(pVCpu);
7297	Assert(!pVCpu->hm.s.Event.fPending);
7298	}
7299
7300	/* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7301	hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7302
7303	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7304	and if we're injecting an event we should have a TRPM trap pending. */
7305	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7306	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
7307	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7308	#endif
7309
7310	/* Save guest state and restore host state bits. */
7311	int rc = hmR0VmxLeaveSession(pVCpu);
7312	AssertRCReturn(rc, rc);
7313	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7314	/* Thread-context hooks are unregistered at this point!!! */
7315
7316	/* Sync recompiler state. */
7317	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7318	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7319	\| CPUM_CHANGED_LDTR
7320	\| CPUM_CHANGED_GDTR
7321	\| CPUM_CHANGED_IDTR
7322	\| CPUM_CHANGED_TR
7323	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
7324	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
7325	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
7326	{
7327	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7328	}
7329
7330	Assert(!pVCpu->hm.s.fClearTrapFlag);
7331
7332	/* Update the exit-to-ring 3 reason. */
7333	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
7334
7335	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7336	if (rcExit != VINF_EM_RAW_INTERRUPT)
7337	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7338
7339	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7340
7341	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7342	VMMRZCallRing3RemoveNotification(pVCpu);
7343	VMMRZCallRing3Enable(pVCpu);
7344
7345	return rc;
7346	}
7347
7348
7349	/**
7350	* VMMRZCallRing3() callback wrapper which saves the guest state before we
7351	* longjump to ring-3 and possibly get preempted.
7352	*
7353	* @returns VBox status code.
7354	* @param pVCpu The cross context virtual CPU structure.
7355	* @param enmOperation The operation causing the ring-3 longjump.
7356	* @param pvUser User argument, currently unused, NULL.
7357	*/
7358	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7359	{
7360	RT_NOREF(pvUser);
7361	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7362	{
7363	/*
7364	* !!! IMPORTANT !!!
7365	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7366	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7367	*/
7368	VMMRZCallRing3RemoveNotification(pVCpu);
7369	VMMRZCallRing3Disable(pVCpu);
7370	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7371	RTThreadPreemptDisable(&PreemptState);
7372
7373	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7374	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7375	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7376
7377	#if HC_ARCH_BITS == 64
7378	/* Restore host-state bits that VT-x only restores partially. */
7379	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7380	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7381	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7382	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7383	#endif
7384
7385	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7386	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7387	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7388
7389	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7390	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7391	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7392	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7393	{
7394	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7395	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7396	}
7397
7398	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7399	VMMR0ThreadCtxHookDisable(pVCpu);
7400	HMR0LeaveCpu(pVCpu);
7401	RTThreadPreemptRestore(&PreemptState);
7402	return VINF_SUCCESS;
7403	}
7404
7405	Assert(pVCpu);
7406	Assert(pvUser);
7407	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7408	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7409
7410	VMMRZCallRing3Disable(pVCpu);
7411	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7412
7413	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7414
7415	int rc = hmR0VmxLongJmpToRing3(pVCpu);
7416	AssertRCReturn(rc, rc);
7417
7418	VMMRZCallRing3Enable(pVCpu);
7419	return VINF_SUCCESS;
7420	}
7421
7422
7423	/**
7424	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7425	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7426	*
7427	* @param pVCpu The cross context virtual CPU structure.
7428	*/
7429	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7430	{
7431	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7432	{
7433	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7434	{
7435	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7436	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7437	AssertRC(rc);
7438	Log4Func(("Setup interrupt-window exiting\n"));
7439	}
7440	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7441	}
7442
7443
7444	/**
7445	* Clears the interrupt-window exiting control in the VMCS.
7446	*
7447	* @param pVCpu The cross context virtual CPU structure.
7448	*/
7449	DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7450	{
7451	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT);
7452	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7453	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7454	AssertRC(rc);
7455	Log4Func(("Cleared interrupt-window exiting\n"));
7456	}
7457
7458
7459	/**
7460	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7461	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7462	*
7463	* @param pVCpu The cross context virtual CPU structure.
7464	*/
7465	DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7466	{
7467	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7468	{
7469	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7470	{
7471	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7472	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7473	AssertRC(rc);
7474	Log4Func(("Setup NMI-window exiting\n"));
7475	}
7476	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7477	}
7478
7479
7480	/**
7481	* Clears the NMI-window exiting control in the VMCS.
7482	*
7483	* @param pVCpu The cross context virtual CPU structure.
7484	*/
7485	DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7486	{
7487	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT);
7488	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7489	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7490	AssertRC(rc);
7491	Log4Func(("Cleared NMI-window exiting\n"));
7492	}
7493
7494
7495	/**
7496	* Evaluates the event to be delivered to the guest and sets it as the pending
7497	* event.
7498	*
7499	* @returns The VT-x guest-interruptibility state.
7500	* @param pVCpu The cross context virtual CPU structure.
7501	*/
7502	static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu)
7503	{
7504	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7505	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7506	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu);
7507	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7508	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7509	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7510
7511	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7512	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7513	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7514	Assert(!TRPMHasTrap(pVCpu));
7515
7516	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7517	APICUpdatePendingInterrupts(pVCpu);
7518
7519	/*
7520	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7521	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7522	*/
7523	/** @todo SMI. SMIs take priority over NMIs. */
7524	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7525	{
7526	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7527	if ( !pVCpu->hm.s.Event.fPending
7528	&& !fBlockNmi
7529	&& !fBlockSti
7530	&& !fBlockMovSS)
7531	{
7532	Log4Func(("Pending NMI\n"));
7533	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INT_INFO_VALID;
7534	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7535
7536	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7537	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7538	}
7539	else
7540	hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7541	}
7542	/*
7543	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7544	* a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7545	*/
7546	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
7547	&& !pVCpu->hm.s.fSingleInstruction)
7548	{
7549	Assert(!DBGFIsStepping(pVCpu));
7550	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7551	AssertRCReturn(rc, 0);
7552	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7553	if ( !pVCpu->hm.s.Event.fPending
7554	&& !fBlockInt
7555	&& !fBlockSti
7556	&& !fBlockMovSS)
7557	{
7558	uint8_t u8Interrupt;
7559	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7560	if (RT_SUCCESS(rc))
7561	{
7562	Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7563	uint32_t u32IntInfo = u8Interrupt
7564	\| VMX_EXIT_INT_INFO_VALID
7565	\| (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7566
7567	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
7568	}
7569	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7570	{
7571	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
7572	hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7573	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7574
7575	/*
7576	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7577	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7578	* need to re-set this force-flag here.
7579	*/
7580	}
7581	else
7582	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7583	}
7584	else
7585	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7586	}
7587
7588	return fIntrState;
7589	}
7590
7591
7592	/**
7593	* Sets a pending-debug exception to be delivered to the guest if the guest is
7594	* single-stepping in the VMCS.
7595	*
7596	* @param pVCpu The cross context virtual CPU structure.
7597	*/
7598	DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7599	{
7600	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7601	RT_NOREF(pVCpu);
7602	return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
7603	}
7604
7605
7606	/**
7607	* Injects any pending events into the guest if the guest is in a state to
7608	* receive them.
7609	*
7610	* @returns Strict VBox status code (i.e. informational status codes too).
7611	* @param pVCpu The cross context virtual CPU structure.
7612	* @param fIntrState The VT-x guest-interruptibility state.
7613	* @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7614	* return VINF_EM_DBG_STEPPED if the event was
7615	* dispatched directly.
7616	*/
7617	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, uint32_t fIntrState, bool fStepping)
7618	{
7619	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7620	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7621
7622	bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7623	bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7624
7625	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7626	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7627	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7628	Assert(!TRPMHasTrap(pVCpu));
7629
7630	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7631	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7632	if (pVCpu->hm.s.Event.fPending)
7633	{
7634	/*
7635	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7636	* pending even while injecting an event and in this case, we want a VM-exit as soon as
7637	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
7638	*
7639	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7640	*/
7641	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7642	#ifdef VBOX_STRICT
7643	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
7644	{
7645	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7646	Assert(!fBlockInt);
7647	Assert(!fBlockSti);
7648	Assert(!fBlockMovSS);
7649	}
7650	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
7651	{
7652	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7653	Assert(!fBlockSti);
7654	Assert(!fBlockMovSS);
7655	Assert(!fBlockNmi);
7656	}
7657	#endif
7658	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7659	uIntType));
7660	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7661	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7662	&fIntrState);
7663	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7664
7665	/* Update the interruptibility-state as it could have been changed by
7666	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7667	fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7668	fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7669
7670	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
7671	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7672	else
7673	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7674	}
7675
7676	/* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7677	if ( fBlockSti
7678	\|\| fBlockMovSS)
7679	{
7680	if (!pVCpu->hm.s.fSingleInstruction)
7681	{
7682	/*
7683	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7684	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7685	* See Intel spec. 27.3.4 "Saving Non-Register State".
7686	*/
7687	Assert(!DBGFIsStepping(pVCpu));
7688	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7689	AssertRCReturn(rc, rc);
7690	if (pCtx->eflags.Bits.u1TF)
7691	{
7692	int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7693	AssertRCReturn(rc2, rc2);
7694	}
7695	}
7696	else if (pCtx->eflags.Bits.u1TF)
7697	{
7698	/*
7699	* We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7700	* BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7701	*/
7702	Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
7703	fIntrState = 0;
7704	}
7705	}
7706
7707	/*
7708	* There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7709	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7710	*/
7711	int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7712	AssertRCReturn(rc3, rc3);
7713
7714	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7715	NOREF(fBlockMovSS); NOREF(fBlockSti);
7716	return rcStrict;
7717	}
7718
7719
7720	/**
7721	* Injects a double-fault (\#DF) exception into the VM.
7722	*
7723	* @returns Strict VBox status code (i.e. informational status codes too).
7724	* @param pVCpu The cross context virtual CPU structure.
7725	* @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7726	* and should return VINF_EM_DBG_STEPPED if the event
7727	* is injected directly (register modified by us, not
7728	* by hardware on VM-entry).
7729	* @param pfIntrState Pointer to the current guest interruptibility-state.
7730	* This interruptibility-state will be updated if
7731	* necessary. This cannot not be NULL.
7732	*/
7733	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, bool fStepping, uint32_t *pfIntrState)
7734	{
7735	uint32_t const u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INT_INFO_VALID
7736	\| (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT)
7737	\| VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7738	return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */, fStepping,
7739	pfIntrState);
7740	}
7741
7742
7743	/**
7744	* Injects a general-protection (\#GP) fault into the VM.
7745	*
7746	* @returns Strict VBox status code (i.e. informational status codes too).
7747	* @param pVCpu The cross context virtual CPU structure.
7748	* @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7749	* mode, i.e. in real-mode it's not valid).
7750	* @param u32ErrorCode The error code associated with the \#GP.
7751	* @param fStepping Whether we're running in
7752	* hmR0VmxRunGuestCodeStep() and should return
7753	* VINF_EM_DBG_STEPPED if the event is injected
7754	* directly (register modified by us, not by
7755	* hardware on VM-entry).
7756	* @param pfIntrState Pointer to the current guest interruptibility-state.
7757	* This interruptibility-state will be updated if
7758	* necessary. This cannot not be NULL.
7759	*/
7760	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, bool fErrorCodeValid, uint32_t u32ErrorCode, bool fStepping,
7761	uint32_t *pfIntrState)
7762	{
7763	uint32_t const u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INT_INFO_VALID
7764	\| (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT)
7765	\| (fErrorCodeValid ? VMX_EXIT_INT_INFO_ERROR_CODE_VALID : 0);
7766	return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */, fStepping,
7767	pfIntrState);
7768	}
7769
7770
7771	/**
7772	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7773	* stack.
7774	*
7775	* @returns Strict VBox status code (i.e. informational status codes too).
7776	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7777	* @param pVCpu The cross context virtual CPU structure.
7778	* @param uValue The value to push to the guest stack.
7779	*/
7780	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
7781	{
7782	/*
7783	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7784	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7785	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7786	*/
7787	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7788	if (pCtx->sp == 1)
7789	return VINF_EM_RESET;
7790	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7791	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
7792	AssertRC(rc);
7793	return rc;
7794	}
7795
7796
7797	/**
7798	* Injects an event into the guest upon VM-entry by updating the relevant fields
7799	* in the VM-entry area in the VMCS.
7800	*
7801	* @returns Strict VBox status code (i.e. informational status codes too).
7802	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7803	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7804	*
7805	* @param pVCpu The cross context virtual CPU structure.
7806	* @param u64IntInfo The VM-entry interruption-information field.
7807	* @param cbInstr The VM-entry instruction length in bytes (for
7808	* software interrupts, exceptions and privileged
7809	* software exceptions).
7810	* @param u32ErrCode The VM-entry exception error code.
7811	* @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7812	* @param pfIntrState Pointer to the current guest interruptibility-state.
7813	* This interruptibility-state will be updated if
7814	* necessary. This cannot not be NULL.
7815	* @param fStepping Whether we're running in
7816	* hmR0VmxRunGuestCodeStep() and should return
7817	* VINF_EM_DBG_STEPPED if the event is injected
7818	* directly (register modified by us, not by
7819	* hardware on VM-entry).
7820	*/
7821	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7822	RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7823	{
7824	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7825	AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7826	Assert(pfIntrState);
7827
7828	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7829	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7830	uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
7831	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
7832
7833	#ifdef VBOX_STRICT
7834	/*
7835	* Validate the error-code-valid bit for hardware exceptions.
7836	* No error codes for exceptions in real-mode.
7837	*
7838	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7839	*/
7840	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7841	&& !CPUMIsGuestInRealModeEx(pCtx))
7842	{
7843	switch (uVector)
7844	{
7845	case X86_XCPT_PF:
7846	case X86_XCPT_DF:
7847	case X86_XCPT_TS:
7848	case X86_XCPT_NP:
7849	case X86_XCPT_SS:
7850	case X86_XCPT_GP:
7851	case X86_XCPT_AC:
7852	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
7853	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7854	RT_FALL_THRU();
7855	default:
7856	break;
7857	}
7858	}
7859	#endif
7860
7861	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
7862	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
7863	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
7864
7865	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7866
7867	/*
7868	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
7869	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7870	* interrupt handler in the (real-mode) guest.
7871	*
7872	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7873	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7874	*/
7875	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7876	{
7877	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7878	{
7879	/*
7880	* For unrestricted execution enabled CPUs running real-mode guests, we must not
7881	* set the deliver-error-code bit.
7882	*
7883	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7884	*/
7885	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
7886	}
7887	else
7888	{
7889	PVM pVM = pVCpu->CTX_SUFF(pVM);
7890	Assert(PDMVmmDevHeapIsEnabled(pVM));
7891	Assert(pVM->hm.s.vmx.pRealModeTSS);
7892
7893	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7894	int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK \| CPUMCTX_EXTRN_RIP
7895	\| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
7896	AssertRCReturn(rc2, rc2);
7897
7898	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7899	size_t const cbIdtEntry = sizeof(X86IDTR16);
7900	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
7901	{
7902	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7903	if (uVector == X86_XCPT_DF)
7904	return VINF_EM_RESET;
7905
7906	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7907	if (uVector == X86_XCPT_GP)
7908	return hmR0VmxInjectXcptDF(pVCpu, fStepping, pfIntrState);
7909
7910	/*
7911	* If we're injecting an event with no valid IDT entry, inject a #GP.
7912	* No error codes for exceptions in real-mode.
7913	*
7914	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7915	*/
7916	return hmR0VmxInjectXcptGP(pVCpu, false /* fErrCodeValid /, 0 / u32ErrCode */, fStepping, pfIntrState);
7917	}
7918
7919	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7920	uint16_t uGuestIp = pCtx->ip;
7921	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
7922	{
7923	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7924	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
7925	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7926	}
7927	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
7928	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7929
7930	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7931	X86IDTR16 IdtEntry;
7932	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
7933	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7934	AssertRCReturn(rc2, rc2);
7935
7936	/* Construct the stack frame for the interrupt/exception handler. */
7937	VBOXSTRICTRC rcStrict;
7938	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
7939	if (rcStrict == VINF_SUCCESS)
7940	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
7941	if (rcStrict == VINF_SUCCESS)
7942	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
7943
7944	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
7945	if (rcStrict == VINF_SUCCESS)
7946	{
7947	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
7948	pCtx->rip = IdtEntry.offSel;
7949	pCtx->cs.Sel = IdtEntry.uSel;
7950	pCtx->cs.ValidSel = IdtEntry.uSel;
7951	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7952	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
7953	&& uVector == X86_XCPT_PF)
7954	pCtx->cr2 = GCPtrFaultAddress;
7955
7956	/* If any other guest-state bits are changed here, make sure to update
7957	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7958	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
7959	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
7960	\| HM_CHANGED_GUEST_RSP);
7961
7962	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7963	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
7964	{
7965	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
7966	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
7967	Log4Func(("Clearing inhibition due to STI\n"));
7968	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
7969	}
7970	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7971	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
7972
7973	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7974	it, if we are returning to ring-3 before executing guest code. */
7975	pVCpu->hm.s.Event.fPending = false;
7976
7977	/* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7978	if (fStepping)
7979	rcStrict = VINF_EM_DBG_STEPPED;
7980	}
7981	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7982	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7983	return rcStrict;
7984	}
7985	}
7986
7987	/* Validate. */
7988	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7989	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
7990
7991	/* Inject. */
7992	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7993	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
7994	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7995	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7996	AssertRCReturn(rc, rc);
7997
7998	/* Update CR2. */
7999	if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8000	&& uVector == X86_XCPT_PF)
8001	pCtx->cr2 = GCPtrFaultAddress;
8002
8003	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8004
8005	return VINF_SUCCESS;
8006	}
8007
8008
8009	/**
8010	* Clears the interrupt-window exiting control in the VMCS and if necessary
8011	* clears the current event in the VMCS as well.
8012	*
8013	* @returns VBox status code.
8014	* @param pVCpu The cross context virtual CPU structure.
8015	*
8016	* @remarks Use this function only to clear events that have not yet been
8017	* delivered to the guest but are injected in the VMCS!
8018	* @remarks No-long-jump zone!!!
8019	*/
8020	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8021	{
8022	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8023	{
8024	hmR0VmxClearIntWindowExitVmcs(pVCpu);
8025	Log4Func(("Cleared interrupt window\n"));
8026	}
8027
8028	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8029	{
8030	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8031	Log4Func(("Cleared NMI window\n"));
8032	}
8033	}
8034
8035
8036	/**
8037	* Enters the VT-x session.
8038	*
8039	* @returns VBox status code.
8040	* @param pVCpu The cross context virtual CPU structure.
8041	* @param pHostCpu Pointer to the global CPU info struct.
8042	*/
8043	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
8044	{
8045	AssertPtr(pVCpu);
8046	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8047	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8048	RT_NOREF(pHostCpu);
8049
8050	LogFlowFunc(("pVCpu=%p\n", pVCpu));
8051	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8052	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8053
8054	#ifdef VBOX_STRICT
8055	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8056	RTCCUINTREG uHostCR4 = ASMGetCR4();
8057	if (!(uHostCR4 & X86_CR4_VMXE))
8058	{
8059	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8060	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8061	}
8062	#endif
8063
8064	/*
8065	* Load the VCPU's VMCS as the current (and active) one.
8066	*/
8067	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8068	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8069	if (RT_FAILURE(rc))
8070	return rc;
8071
8072	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8073	pVCpu->hm.s.fLeaveDone = false;
8074	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8075
8076	return VINF_SUCCESS;
8077	}
8078
8079
8080	/**
8081	* The thread-context callback (only on platforms which support it).
8082	*
8083	* @param enmEvent The thread-context event.
8084	* @param pVCpu The cross context virtual CPU structure.
8085	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8086	* @thread EMT(pVCpu)
8087	*/
8088	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8089	{
8090	NOREF(fGlobalInit);
8091
8092	switch (enmEvent)
8093	{
8094	case RTTHREADCTXEVENT_OUT:
8095	{
8096	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8097	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8098	VMCPU_ASSERT_EMT(pVCpu);
8099
8100	/* No longjmps (logger flushes, locks) in this fragile context. */
8101	VMMRZCallRing3Disable(pVCpu);
8102	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8103
8104	/*
8105	* Restore host-state (FPU, debug etc.)
8106	*/
8107	if (!pVCpu->hm.s.fLeaveDone)
8108	{
8109	/*
8110	* Do -not- import the guest-state here as we might already be in the middle of importing
8111	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8112	*/
8113	hmR0VmxLeave(pVCpu, false /* fImportState */);
8114	pVCpu->hm.s.fLeaveDone = true;
8115	}
8116
8117	/* Leave HM context, takes care of local init (term). */
8118	int rc = HMR0LeaveCpu(pVCpu);
8119	AssertRC(rc); NOREF(rc);
8120
8121	/* Restore longjmp state. */
8122	VMMRZCallRing3Enable(pVCpu);
8123	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8124	break;
8125	}
8126
8127	case RTTHREADCTXEVENT_IN:
8128	{
8129	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8130	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8131	VMCPU_ASSERT_EMT(pVCpu);
8132
8133	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8134	VMMRZCallRing3Disable(pVCpu);
8135	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8136
8137	/* Initialize the bare minimum state required for HM. This takes care of
8138	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8139	int rc = hmR0EnterCpu(pVCpu);
8140	AssertRC(rc);
8141	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8142	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8143
8144	/* Load the active VMCS as the current one. */
8145	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8146	{
8147	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8148	AssertRC(rc); NOREF(rc);
8149	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8150	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8151	}
8152	pVCpu->hm.s.fLeaveDone = false;
8153
8154	/* Restore longjmp state. */
8155	VMMRZCallRing3Enable(pVCpu);
8156	break;
8157	}
8158
8159	default:
8160	break;
8161	}
8162	}
8163
8164
8165	/**
8166	* Exports the host state into the VMCS host-state area.
8167	* Sets up the VM-exit MSR-load area.
8168	*
8169	* The CPU state will be loaded from these fields on every successful VM-exit.
8170	*
8171	* @returns VBox status code.
8172	* @param pVCpu The cross context virtual CPU structure.
8173	*
8174	* @remarks No-long-jump zone!!!
8175	*/
8176	static int hmR0VmxExportHostState(PVMCPU pVCpu)
8177	{
8178	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8179
8180	int rc = VINF_SUCCESS;
8181	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8182	{
8183	rc = hmR0VmxExportHostControlRegs();
8184	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8185
8186	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8187	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8188
8189	rc = hmR0VmxExportHostMsrs(pVCpu);
8190	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8191
8192	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8193	}
8194	return rc;
8195	}
8196
8197
8198	/**
8199	* Saves the host state in the VMCS host-state.
8200	*
8201	* @returns VBox status code.
8202	* @param pVCpu The cross context virtual CPU structure.
8203	*
8204	* @remarks No-long-jump zone!!!
8205	*/
8206	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
8207	{
8208	AssertPtr(pVCpu);
8209	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8210
8211	/*
8212	* Export the host state here while entering HM context.
8213	* When thread-context hooks are used, we might get preempted and have to re-save the host
8214	* state but most of the time we won't be, so do it here before we disable interrupts.
8215	*/
8216	return hmR0VmxExportHostState(pVCpu);
8217	}
8218
8219
8220	/**
8221	* Exports the guest state into the VMCS guest-state area.
8222	*
8223	* The will typically be done before VM-entry when the guest-CPU state and the
8224	* VMCS state may potentially be out of sync.
8225	*
8226	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8227	* VM-entry controls.
8228	* Sets up the appropriate VMX non-root function to execute guest code based on
8229	* the guest CPU mode.
8230	*
8231	* @returns VBox strict status code.
8232	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8233	* without unrestricted guest access and the VMMDev is not presently
8234	* mapped (e.g. EFI32).
8235	*
8236	* @param pVCpu The cross context virtual CPU structure.
8237	*
8238	* @remarks No-long-jump zone!!!
8239	*/
8240	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu)
8241	{
8242	AssertPtr(pVCpu);
8243	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8244
8245	LogFlowFunc(("pVCpu=%p\n", pVCpu));
8246
8247	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
8248
8249	/* Determine real-on-v86 mode. */
8250	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8251	if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
8252	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
8253	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8254
8255	/*
8256	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8257	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8258	*/
8259	int rc = hmR0VmxSelectVMRunHandler(pVCpu);
8260	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8261
8262	/* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8263	rc = hmR0VmxExportGuestEntryCtls(pVCpu);
8264	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8265
8266	/* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8267	rc = hmR0VmxExportGuestExitCtls(pVCpu);
8268	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8269
8270	rc = hmR0VmxExportGuestCR0(pVCpu);
8271	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8272
8273	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu);
8274	if (rcStrict == VINF_SUCCESS)
8275	{ /* likely */ }
8276	else
8277	{
8278	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
8279	return rcStrict;
8280	}
8281
8282	rc = hmR0VmxExportGuestSegmentRegs(pVCpu);
8283	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8284
8285	/* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8286	may alter controls if we determine we don't have to swap EFER after all. */
8287	rc = hmR0VmxExportGuestMsrs(pVCpu);
8288	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8289
8290	rc = hmR0VmxExportGuestApicTpr(pVCpu);
8291	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8292
8293	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8294	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8295
8296	/* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8297	not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8298	rc = hmR0VmxExportGuestRip(pVCpu);
8299	rc \|= hmR0VmxExportGuestRsp(pVCpu);
8300	rc \|= hmR0VmxExportGuestRflags(pVCpu);
8301	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8302
8303	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8304	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8305	\| HM_CHANGED_GUEST_CR2
8306	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8307	\| HM_CHANGED_GUEST_X87
8308	\| HM_CHANGED_GUEST_SSE_AVX
8309	\| HM_CHANGED_GUEST_OTHER_XSAVE
8310	\| HM_CHANGED_GUEST_XCRx
8311	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8312	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8313	\| HM_CHANGED_GUEST_TSC_AUX
8314	\| HM_CHANGED_GUEST_OTHER_MSRS
8315	\| HM_CHANGED_GUEST_HWVIRT
8316	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8317
8318	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8319	return rc;
8320	}
8321
8322
8323	/**
8324	* Exports the state shared between the host and guest into the VMCS.
8325	*
8326	* @param pVCpu The cross context virtual CPU structure.
8327	*
8328	* @remarks No-long-jump zone!!!
8329	*/
8330	static void hmR0VmxExportSharedState(PVMCPU pVCpu)
8331	{
8332	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8333	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8334
8335	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8336	{
8337	int rc = hmR0VmxExportSharedDebugState(pVCpu);
8338	AssertRC(rc);
8339	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8340
8341	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8342	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8343	{
8344	rc = hmR0VmxExportGuestRflags(pVCpu);
8345	AssertRC(rc);
8346	}
8347	}
8348
8349	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8350	{
8351	hmR0VmxLazyLoadGuestMsrs(pVCpu);
8352	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8353	}
8354
8355	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8356	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8357	}
8358
8359
8360	/**
8361	* Worker for loading the guest-state bits in the inner VT-x execution loop.
8362	*
8363	* @returns Strict VBox status code (i.e. informational status codes too).
8364	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8365	* without unrestricted guest access and the VMMDev is not presently
8366	* mapped (e.g. EFI32).
8367	*
8368	* @param pVCpu The cross context virtual CPU structure.
8369	*
8370	* @remarks No-long-jump zone!!!
8371	*/
8372	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu)
8373	{
8374	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8375	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8376	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8377
8378	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8379	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8380	#endif
8381
8382	/*
8383	* For many exits it's only RIP that changes and hence try to export it first
8384	* without going through a lot of change flag checks.
8385	*/
8386	VBOXSTRICTRC rcStrict;
8387	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8388	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8389	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8390	{
8391	rcStrict = hmR0VmxExportGuestRip(pVCpu);
8392	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8393	{ /* likely */}
8394	else
8395	AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8396	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8397	}
8398	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8399	{
8400	rcStrict = hmR0VmxExportGuestState(pVCpu);
8401	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8402	{ /* likely */}
8403	else
8404	{
8405	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8406	VBOXSTRICTRC_VAL(rcStrict)));
8407	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8408	return rcStrict;
8409	}
8410	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8411	}
8412	else
8413	rcStrict = VINF_SUCCESS;
8414
8415	#ifdef VBOX_STRICT
8416	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8417	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8418	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8419	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8420	("fCtxChanged=%#RX64\n", fCtxChanged));
8421	#endif
8422	return rcStrict;
8423	}
8424
8425
8426	/**
8427	* Does the preparations before executing guest code in VT-x.
8428	*
8429	* This may cause longjmps to ring-3 and may even result in rescheduling to the
8430	* recompiler/IEM. We must be cautious what we do here regarding committing
8431	* guest-state information into the VMCS assuming we assuredly execute the
8432	* guest in VT-x mode.
8433	*
8434	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
8435	* the common-state (TRPM/forceflags), we must undo those changes so that the
8436	* recompiler/IEM can (and should) use them when it resumes guest execution.
8437	* Otherwise such operations must be done when we can no longer exit to ring-3.
8438	*
8439	* @returns Strict VBox status code (i.e. informational status codes too).
8440	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8441	* have been disabled.
8442	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8443	* double-fault into the guest.
8444	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8445	* dispatched directly.
8446	* @retval VINF_* scheduling changes, we have to go back to ring-3.
8447	*
8448	* @param pVCpu The cross context virtual CPU structure.
8449	* @param pVmxTransient Pointer to the VMX transient structure.
8450	* @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8451	* us ignore some of the reasons for returning to
8452	* ring-3, and return VINF_EM_DBG_STEPPED if event
8453	* dispatching took place.
8454	*/
8455	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
8456	{
8457	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8458
8459	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_ONLY_IN_IEM
8460	Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
8461	return VINF_EM_RESCHEDULE_REM;
8462	#endif
8463
8464	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8465	PGMRZDynMapFlushAutoSet(pVCpu);
8466	#endif
8467
8468	/* Check force flag actions that might require us to go back to ring-3. */
8469	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
8470	if (rcStrict == VINF_SUCCESS)
8471	{ /* FFs doesn't get set all the time. */ }
8472	else
8473	return rcStrict;
8474
8475	/*
8476	* Setup the virtualized-APIC accesses.
8477	*
8478	* Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8479	* in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8480	*
8481	* This is the reason we do it here and not in hmR0VmxExportGuestState().
8482	*/
8483	PVM pVM = pVCpu->CTX_SUFF(pVM);
8484	if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8485	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
8486	&& PDMHasApic(pVM))
8487	{
8488	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8489	Assert(u64MsrApicBase);
8490	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8491
8492	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8493
8494	/* Unalias any existing mapping. */
8495	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8496	AssertRCReturn(rc, rc);
8497
8498	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8499	Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8500	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
8501	AssertRCReturn(rc, rc);
8502
8503	/* Update the per-VCPU cache of the APIC base MSR. */
8504	pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8505	}
8506
8507	if (TRPMHasTrap(pVCpu))
8508	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8509	uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu);
8510
8511	/*
8512	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8513	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8514	* also result in triple-faulting the VM.
8515	*/
8516	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, fIntrState, fStepping);
8517	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8518	{ /* likely */ }
8519	else
8520	{
8521	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8522	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8523	return rcStrict;
8524	}
8525
8526	/*
8527	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
8528	* import CR3 themselves. We will need to update them here, as even as late as the above
8529	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8530	* the below force flags to be set.
8531	*/
8532	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8533	{
8534	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
8535	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8536	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
8537	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8538	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8539	}
8540	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8541	{
8542	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8543	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8544	}
8545
8546	/*
8547	* No longjmps to ring-3 from this point on!!!
8548	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8549	* This also disables flushing of the R0-logger instance (if any).
8550	*/
8551	VMMRZCallRing3Disable(pVCpu);
8552
8553	/*
8554	* Export the guest state bits.
8555	*
8556	* We cannot perform longjmps while loading the guest state because we do not preserve the
8557	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
8558	* CPU migration.
8559	*
8560	* If we are injecting events to a real-on-v86 mode guest, we will have to update
8561	* RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8562	* Hence, loading of the guest state needs to be done -after- injection of events.
8563	*/
8564	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu);
8565	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8566	{ /* likely */ }
8567	else
8568	{
8569	VMMRZCallRing3Enable(pVCpu);
8570	return rcStrict;
8571	}
8572
8573	/*
8574	* We disable interrupts so that we don't miss any interrupts that would flag preemption
8575	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8576	* preemption disabled for a while. Since this is purly to aid the
8577	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8578	* disable interrupt on NT.
8579	*
8580	* We need to check for force-flags that could've possible been altered since we last
8581	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8582	* see @bugref{6398}).
8583	*
8584	* We also check a couple of other force-flags as a last opportunity to get the EMT back
8585	* to ring-3 before executing guest code.
8586	*/
8587	pVmxTransient->fEFlags = ASMIntDisableFlags();
8588
8589	if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
8590	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8591	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8592	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
8593	{
8594	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8595	{
8596	pVCpu->hm.s.Event.fPending = false;
8597
8598	/*
8599	* We've injected any pending events. This is really the point of no return (to ring-3).
8600	*
8601	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
8602	* returns from this function, so don't enable them here.
8603	*/
8604	return VINF_SUCCESS;
8605	}
8606
8607	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
8608	rcStrict = VINF_EM_RAW_INTERRUPT;
8609	}
8610	else
8611	{
8612	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8613	rcStrict = VINF_EM_RAW_TO_R3;
8614	}
8615
8616	ASMSetFlags(pVmxTransient->fEFlags);
8617	VMMRZCallRing3Enable(pVCpu);
8618
8619	return rcStrict;
8620	}
8621
8622
8623	/**
8624	* Prepares to run guest code in VT-x and we've committed to doing so. This
8625	* means there is no backing out to ring-3 or anywhere else at this
8626	* point.
8627	*
8628	* @param pVCpu The cross context virtual CPU structure.
8629	* @param pVmxTransient Pointer to the VMX transient structure.
8630	*
8631	* @remarks Called with preemption disabled.
8632	* @remarks No-long-jump zone!!!
8633	*/
8634	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
8635	{
8636	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8637	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8638	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8639
8640	/*
8641	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8642	*/
8643	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8644	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8645
8646	PVM pVM = pVCpu->CTX_SUFF(pVM);
8647	if (!CPUMIsGuestFPUStateActive(pVCpu))
8648	{
8649	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8650	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8651	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
8652	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8653	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8654	}
8655
8656	/*
8657	* Lazy-update of the host MSRs values in the auto-load/store MSR area.
8658	*/
8659	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8660	&& pVCpu->hm.s.vmx.cMsrs > 0)
8661	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8662
8663	/*
8664	* Re-save the host state bits as we may've been preempted (only happens when
8665	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8666	* Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8667	* if we change the switcher back to 32-bit, we must save the 32-bit host state here.
8668	* See @bugref{8432}.
8669	*/
8670	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8671	{
8672	int rc = hmR0VmxExportHostState(pVCpu);
8673	AssertRC(rc);
8674	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8675	}
8676	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8677
8678	/*
8679	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
8680	*/
8681	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8682	hmR0VmxExportSharedState(pVCpu);
8683	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8684
8685	/* Store status of the shared guest-host state at the time of VM-entry. */
8686	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8687	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8688	{
8689	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8690	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8691	}
8692	else
8693	#endif
8694	{
8695	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8696	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8697	}
8698
8699	/*
8700	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8701	*/
8702	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8703	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8704
8705	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8706	RTCPUID idCurrentCpu = pCpu->idCpu;
8707	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8708	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
8709	{
8710	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8711	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8712	}
8713
8714	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8715	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8716	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8717	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8718
8719	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8720
8721	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8722	to start executing. */
8723
8724	/*
8725	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8726	*/
8727	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
8728	{
8729	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
8730	{
8731	bool fMsrUpdated;
8732	hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8733	int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8734	&fMsrUpdated);
8735	AssertRC(rc2);
8736	Assert(fMsrUpdated \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8737	/* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8738	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8739	}
8740	else
8741	{
8742	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8743	Assert(!pVCpu->hm.s.vmx.cMsrs \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8744	}
8745	}
8746
8747	if (pVM->cpum.ro.GuestFeatures.fIbrs)
8748	{
8749	bool fMsrUpdated;
8750	hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8751	int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8752	&fMsrUpdated);
8753	AssertRC(rc2);
8754	Assert(fMsrUpdated \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8755	/* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8756	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8757	}
8758
8759	#ifdef VBOX_STRICT
8760	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8761	hmR0VmxCheckHostEferMsr(pVCpu);
8762	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8763	#endif
8764	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8765	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
8766	{
8767	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
8768	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8769	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8770	}
8771	#endif
8772	}
8773
8774
8775	/**
8776	* Performs some essential restoration of state after running guest code in
8777	* VT-x.
8778	*
8779	* @param pVCpu The cross context virtual CPU structure.
8780	* @param pVmxTransient Pointer to the VMX transient structure.
8781	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8782	*
8783	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
8784	*
8785	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
8786	* unconditionally when it is safe to do so.
8787	*/
8788	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8789	{
8790	uint64_t const uHostTsc = ASMReadTSC();
8791	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8792
8793	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8794	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8795	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8796	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8797	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8798	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8799
8800	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
8801	TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8802
8803	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8804	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8805	Assert(!ASMIntAreEnabled());
8806	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8807
8808	#if HC_ARCH_BITS == 64
8809	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8810	#endif
8811	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8812	/* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8813	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8814	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8815	#else
8816	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8817	#endif
8818	#ifdef VBOX_STRICT
8819	hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8820	#endif
8821	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8822
8823	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8824	uint32_t uExitReason;
8825	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8826	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8827	AssertRC(rc);
8828	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
8829	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
8830
8831	if (rcVMRun == VINF_SUCCESS)
8832	{
8833	/*
8834	* Update the VM-exit history array here even if the VM-entry failed due to:
8835	* - Invalid guest state.
8836	* - MSR loading.
8837	* - Machine-check event.
8838	*
8839	* In any of the above cases we will still have a "valid" VM-exit reason
8840	* despite @a fVMEntryFailed being false.
8841	*
8842	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8843	*
8844	* Note! We don't have CS or RIP at this point. Will probably address that later
8845	* by amending the history entry added here.
8846	*/
8847	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8848	UINT64_MAX, uHostTsc);
8849
8850	if (!pVmxTransient->fVMEntryFailed)
8851	{
8852	VMMRZCallRing3Enable(pVCpu);
8853
8854	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8855	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8856
8857	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8858	rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8859	AssertRC(rc);
8860	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8861	rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8862	AssertRC(rc);
8863	#else
8864	/*
8865	* Import the guest-interruptibility state always as we need it while evaluating
8866	* injecting events on re-entry.
8867	*
8868	* We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8869	* checking for real-mode while exporting the state because all bits that cause
8870	* mode changes wrt CR0 are intercepted.
8871	*/
8872	rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8873	AssertRC(rc);
8874	#endif
8875
8876	/*
8877	* Sync the TPR shadow with our APIC state.
8878	*/
8879	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8880	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8881	{
8882	rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8883	AssertRC(rc);
8884	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8885	}
8886
8887	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8888	return;
8889	}
8890	}
8891	else
8892	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8893
8894	VMMRZCallRing3Enable(pVCpu);
8895	}
8896
8897
8898	/**
8899	* Runs the guest code using VT-x the normal way.
8900	*
8901	* @returns VBox status code.
8902	* @param pVCpu The cross context virtual CPU structure.
8903	*
8904	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
8905	*/
8906	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu)
8907	{
8908	VMXTRANSIENT VmxTransient;
8909	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8910	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8911	uint32_t cLoops = 0;
8912
8913	for (;; cLoops++)
8914	{
8915	Assert(!HMR0SuspendPending());
8916	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8917
8918	/* Preparatory work for running guest code, this may force us to return
8919	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8920	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8921	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
8922	if (rcStrict != VINF_SUCCESS)
8923	break;
8924
8925	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
8926	int rcRun = hmR0VmxRunGuest(pVCpu);
8927
8928	/* Restore any residual host-state and save any bits shared between host
8929	and guest into the guest-CPU state. Re-enables interrupts! */
8930	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8931
8932	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8933	if (RT_SUCCESS(rcRun))
8934	{ /* very likely */ }
8935	else
8936	{
8937	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8938	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
8939	return rcRun;
8940	}
8941
8942	/* Profile the VM-exit. */
8943	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8944	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8945	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8946	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8947	HMVMX_START_EXIT_DISPATCH_PROF();
8948
8949	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
8950
8951	/* Handle the VM-exit. */
8952	#ifdef HMVMX_USE_FUNCTION_TABLE
8953	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
8954	#else
8955	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient, VmxTransient.uExitReason);
8956	#endif
8957	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8958	if (rcStrict == VINF_SUCCESS)
8959	{
8960	if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
8961	continue; /* likely */
8962	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8963	rcStrict = VINF_EM_RAW_INTERRUPT;
8964	}
8965	break;
8966	}
8967
8968	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8969	return rcStrict;
8970	}
8971
8972
8973
8974	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8975	* probes.
8976	*
8977	* The following few functions and associated structure contains the bloat
8978	* necessary for providing detailed debug events and dtrace probes as well as
8979	* reliable host side single stepping. This works on the principle of
8980	* "subclassing" the normal execution loop and workers. We replace the loop
8981	* method completely and override selected helpers to add necessary adjustments
8982	* to their core operation.
8983	*
8984	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8985	* any performance for debug and analysis features.
8986	*
8987	* @{
8988	*/
8989
8990	/**
8991	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
8992	* the debug run loop.
8993	*/
8994	typedef struct VMXRUNDBGSTATE
8995	{
8996	/** The RIP we started executing at. This is for detecting that we stepped. */
8997	uint64_t uRipStart;
8998	/** The CS we started executing with. */
8999	uint16_t uCsStart;
9000
9001	/** Whether we've actually modified the 1st execution control field. */
9002	bool fModifiedProcCtls : 1;
9003	/** Whether we've actually modified the 2nd execution control field. */
9004	bool fModifiedProcCtls2 : 1;
9005	/** Whether we've actually modified the exception bitmap. */
9006	bool fModifiedXcptBitmap : 1;
9007
9008	/** We desire the modified the CR0 mask to be cleared. */
9009	bool fClearCr0Mask : 1;
9010	/** We desire the modified the CR4 mask to be cleared. */
9011	bool fClearCr4Mask : 1;
9012	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9013	uint32_t fCpe1Extra;
9014	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9015	uint32_t fCpe1Unwanted;
9016	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9017	uint32_t fCpe2Extra;
9018	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9019	uint32_t bmXcptExtra;
9020	/** The sequence number of the Dtrace provider settings the state was
9021	* configured against. */
9022	uint32_t uDtraceSettingsSeqNo;
9023	/** VM-exits to check (one bit per VM-exit). */
9024	uint32_t bmExitsToCheck[3];
9025
9026	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9027	uint32_t fProcCtlsInitial;
9028	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9029	uint32_t fProcCtls2Initial;
9030	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9031	uint32_t bmXcptInitial;
9032	} VMXRUNDBGSTATE;
9033	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9034	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9035
9036
9037	/**
9038	* Initializes the VMXRUNDBGSTATE structure.
9039	*
9040	* @param pVCpu The cross context virtual CPU structure of the
9041	* calling EMT.
9042	* @param pDbgState The structure to initialize.
9043	*/
9044	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9045	{
9046	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
9047	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
9048
9049	pDbgState->fModifiedProcCtls = false;
9050	pDbgState->fModifiedProcCtls2 = false;
9051	pDbgState->fModifiedXcptBitmap = false;
9052	pDbgState->fClearCr0Mask = false;
9053	pDbgState->fClearCr4Mask = false;
9054	pDbgState->fCpe1Extra = 0;
9055	pDbgState->fCpe1Unwanted = 0;
9056	pDbgState->fCpe2Extra = 0;
9057	pDbgState->bmXcptExtra = 0;
9058	pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9059	pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9060	pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9061	}
9062
9063
9064	/**
9065	* Updates the VMSC fields with changes requested by @a pDbgState.
9066	*
9067	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9068	* immediately before executing guest code, i.e. when interrupts are disabled.
9069	* We don't check status codes here as we cannot easily assert or return in the
9070	* latter case.
9071	*
9072	* @param pVCpu The cross context virtual CPU structure.
9073	* @param pDbgState The debug state.
9074	*/
9075	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9076	{
9077	/*
9078	* Ensure desired flags in VMCS control fields are set.
9079	* (Ignoring write failure here, as we're committed and it's just debug extras.)
9080	*
9081	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9082	* there should be no stale data in pCtx at this point.
9083	*/
9084	if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9085	\|\| (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9086	{
9087	pVCpu->hm.s.vmx.u32ProcCtls \|= pDbgState->fCpe1Extra;
9088	pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9089	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9090	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9091	pDbgState->fModifiedProcCtls = true;
9092	}
9093
9094	if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9095	{
9096	pVCpu->hm.s.vmx.u32ProcCtls2 \|= pDbgState->fCpe2Extra;
9097	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9098	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9099	pDbgState->fModifiedProcCtls2 = true;
9100	}
9101
9102	if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9103	{
9104	pVCpu->hm.s.vmx.u32XcptBitmap \|= pDbgState->bmXcptExtra;
9105	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9106	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9107	pDbgState->fModifiedXcptBitmap = true;
9108	}
9109
9110	if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32Cr0Mask != 0)
9111	{
9112	pVCpu->hm.s.vmx.u32Cr0Mask = 0;
9113	VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9114	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9115	}
9116
9117	if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32Cr4Mask != 0)
9118	{
9119	pVCpu->hm.s.vmx.u32Cr4Mask = 0;
9120	VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9121	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9122	}
9123	}
9124
9125
9126	/**
9127	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
9128	* re-entry next time around.
9129	*
9130	* @returns Strict VBox status code (i.e. informational status codes too).
9131	* @param pVCpu The cross context virtual CPU structure.
9132	* @param pDbgState The debug state.
9133	* @param rcStrict The return code from executing the guest using single
9134	* stepping.
9135	*/
9136	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9137	{
9138	/*
9139	* Restore VM-exit control settings as we may not reenter this function the
9140	* next time around.
9141	*/
9142	/* We reload the initial value, trigger what we can of recalculations the
9143	next time around. From the looks of things, that's all that's required atm. */
9144	if (pDbgState->fModifiedProcCtls)
9145	{
9146	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9147	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9148	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9149	AssertRCReturn(rc2, rc2);
9150	pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9151	}
9152
9153	/* We're currently the only ones messing with this one, so just restore the
9154	cached value and reload the field. */
9155	if ( pDbgState->fModifiedProcCtls2
9156	&& pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9157	{
9158	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9159	AssertRCReturn(rc2, rc2);
9160	pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9161	}
9162
9163	/* If we've modified the exception bitmap, we restore it and trigger
9164	reloading and partial recalculation the next time around. */
9165	if (pDbgState->fModifiedXcptBitmap)
9166	pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9167
9168	return rcStrict;
9169	}
9170
9171
9172	/**
9173	* Configures VM-exit controls for current DBGF and DTrace settings.
9174	*
9175	* This updates @a pDbgState and the VMCS execution control fields to reflect
9176	* the necessary VM-exits demanded by DBGF and DTrace.
9177	*
9178	* @param pVCpu The cross context virtual CPU structure.
9179	* @param pDbgState The debug state.
9180	* @param pVmxTransient Pointer to the VMX transient structure. May update
9181	* fUpdateTscOffsettingAndPreemptTimer.
9182	*/
9183	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9184	{
9185	/*
9186	* Take down the dtrace serial number so we can spot changes.
9187	*/
9188	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9189	ASMCompilerBarrier();
9190
9191	/*
9192	* We'll rebuild most of the middle block of data members (holding the
9193	* current settings) as we go along here, so start by clearing it all.
9194	*/
9195	pDbgState->bmXcptExtra = 0;
9196	pDbgState->fCpe1Extra = 0;
9197	pDbgState->fCpe1Unwanted = 0;
9198	pDbgState->fCpe2Extra = 0;
9199	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9200	pDbgState->bmExitsToCheck[i] = 0;
9201
9202	/*
9203	* Software interrupts (INT XXh) - no idea how to trigger these...
9204	*/
9205	PVM pVM = pVCpu->CTX_SUFF(pVM);
9206	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9207	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
9208	{
9209	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9210	}
9211
9212	/*
9213	* INT3 breakpoints - triggered by #BP exceptions.
9214	*/
9215	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9216	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
9217
9218	/*
9219	* Exception bitmap and XCPT events+probes.
9220	*/
9221	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9222	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9223	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
9224
9225	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
9226	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
9227	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
9228	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
9229	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
9230	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
9231	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
9232	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
9233	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
9234	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
9235	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
9236	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
9237	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
9238	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
9239	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
9240	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
9241	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
9242	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
9243
9244	if (pDbgState->bmXcptExtra)
9245	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9246
9247	/*
9248	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9249	*
9250	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
9251	* So, when adding/changing/removing please don't forget to update it.
9252	*
9253	* Some of the macros are picking up local variables to save horizontal space,
9254	* (being able to see it in a table is the lesser evil here).
9255	*/
9256	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9257	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9258	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9259	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9260	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9261	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9262	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9263	} else do { } while (0)
9264	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9265	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9266	{ \
9267	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
9268	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9269	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9270	} else do { } while (0)
9271	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9272	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9273	{ \
9274	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
9275	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9276	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9277	} else do { } while (0)
9278	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9279	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9280	{ \
9281	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
9282	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9283	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9284	} else do { } while (0)
9285
9286	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9287	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9288	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9289	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9290	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9291
9292	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9293	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9294	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9295	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9296	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
9297	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9298	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9299	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9300	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
9301	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9302	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
9303	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9304	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
9305	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9306	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9307	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9308	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9309	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9310	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9311	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9312	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9313	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9314	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9315	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9316	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9317	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9318	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9319	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9320	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9321	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9322	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9323	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9324	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9325	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9326	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9327	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9328
9329	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9330	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9331	{
9332	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_APIC_TPR);
9333	AssertRC(rc);
9334
9335	#if 0 /** @todo fix me */
9336	pDbgState->fClearCr0Mask = true;
9337	pDbgState->fClearCr4Mask = true;
9338	#endif
9339	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9340	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
9341	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9342	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
9343	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
9344	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9345	require clearing here and in the loop if we start using it. */
9346	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9347	}
9348	else
9349	{
9350	if (pDbgState->fClearCr0Mask)
9351	{
9352	pDbgState->fClearCr0Mask = false;
9353	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9354	}
9355	if (pDbgState->fClearCr4Mask)
9356	{
9357	pDbgState->fClearCr4Mask = false;
9358	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9359	}
9360	}
9361	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9362	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9363
9364	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9365	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9366	{
9367	/** @todo later, need to fix handler as it assumes this won't usually happen. */
9368	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9369	}
9370	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9371	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9372
9373	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
9374	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9375	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
9376	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9377	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
9378	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9379	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
9380	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9381	#if 0 /** @todo too slow, fix handler. */
9382	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
9383	#endif
9384	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9385
9386	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9387	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9388	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9389	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9390	{
9391	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
9392	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
9393	}
9394	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9395	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9396	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9397	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9398
9399	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9400	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
9401	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9402	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
9403	{
9404	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
9405	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
9406	}
9407	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
9408	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
9409	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
9410	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
9411
9412	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9413	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9414	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
9415	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9416	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9417	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9418	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
9419	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9420	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9421	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9422	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
9423	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9424	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
9425	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9426	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9427	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9428	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
9429	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9430	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9431	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9432	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9433	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9434
9435	#undef IS_EITHER_ENABLED
9436	#undef SET_ONLY_XBM_IF_EITHER_EN
9437	#undef SET_CPE1_XBM_IF_EITHER_EN
9438	#undef SET_CPEU_XBM_IF_EITHER_EN
9439	#undef SET_CPE2_XBM_IF_EITHER_EN
9440
9441	/*
9442	* Sanitize the control stuff.
9443	*/
9444	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
9445	if (pDbgState->fCpe2Extra)
9446	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
9447	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
9448	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0;
9449	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
9450	{
9451	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9452	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9453	}
9454
9455	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9456	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9457	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9458	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9459	}
9460
9461
9462	/**
9463	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
9464	* appropriate.
9465	*
9466	* The caller has checked the VM-exit against the
9467	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9468	* already, so we don't have to do that either.
9469	*
9470	* @returns Strict VBox status code (i.e. informational status codes too).
9471	* @param pVCpu The cross context virtual CPU structure.
9472	* @param pVmxTransient Pointer to the VMX-transient structure.
9473	* @param uExitReason The VM-exit reason.
9474	*
9475	* @remarks The name of this function is displayed by dtrace, so keep it short
9476	* and to the point. No longer than 33 chars long, please.
9477	*/
9478	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9479	{
9480	/*
9481	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9482	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
9483	*
9484	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9485	* does. Must add/change/remove both places. Same ordering, please.
9486	*
9487	* Added/removed events must also be reflected in the next section
9488	* where we dispatch dtrace events.
9489	*/
9490	bool fDtrace1 = false;
9491	bool fDtrace2 = false;
9492	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9493	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9494	uint32_t uEventArg = 0;
9495	#define SET_EXIT(a_EventSubName) \
9496	do { \
9497	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9498	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9499	} while (0)
9500	#define SET_BOTH(a_EventSubName) \
9501	do { \
9502	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9503	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9504	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9505	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9506	} while (0)
9507	switch (uExitReason)
9508	{
9509	case VMX_EXIT_MTF:
9510	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9511
9512	case VMX_EXIT_XCPT_OR_NMI:
9513	{
9514	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9515	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
9516	{
9517	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
9518	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
9519	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
9520	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9521	{
9522	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
9523	{
9524	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9525	uEventArg = pVmxTransient->uExitIntErrorCode;
9526	}
9527	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9528	switch (enmEvent1)
9529	{
9530	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9531	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9532	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9533	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9534	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9535	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9536	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9537	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9538	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9539	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9540	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9541	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9542	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9543	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9544	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9545	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9546	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9547	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9548	default: break;
9549	}
9550	}
9551	else
9552	AssertFailed();
9553	break;
9554
9555	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
9556	uEventArg = idxVector;
9557	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9558	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9559	break;
9560	}
9561	break;
9562	}
9563
9564	case VMX_EXIT_TRIPLE_FAULT:
9565	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9566	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9567	break;
9568	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9569	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9570	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9571	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9572	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9573
9574	/* Instruction specific VM-exits: */
9575	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9576	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9577	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9578	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9579	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9580	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9581	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9582	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9583	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9584	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9585	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9586	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9587	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9588	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9589	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9590	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9591	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9592	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9593	case VMX_EXIT_MOV_CRX:
9594	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9595	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9596	SET_BOTH(CRX_READ);
9597	else
9598	SET_BOTH(CRX_WRITE);
9599	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
9600	break;
9601	case VMX_EXIT_MOV_DRX:
9602	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9603	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
9604	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9605	SET_BOTH(DRX_READ);
9606	else
9607	SET_BOTH(DRX_WRITE);
9608	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
9609	break;
9610	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9611	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9612	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9613	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9614	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9615	case VMX_EXIT_GDTR_IDTR_ACCESS:
9616	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9617	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
9618	{
9619	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9620	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9621	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9622	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9623	}
9624	break;
9625
9626	case VMX_EXIT_LDTR_TR_ACCESS:
9627	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9628	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
9629	{
9630	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9631	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9632	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9633	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9634	}
9635	break;
9636
9637	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9638	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9639	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9640	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9641	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9642	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9643	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9644	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9645	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9646	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9647	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9648
9649	/* Events that aren't relevant at this point. */
9650	case VMX_EXIT_EXT_INT:
9651	case VMX_EXIT_INT_WINDOW:
9652	case VMX_EXIT_NMI_WINDOW:
9653	case VMX_EXIT_TPR_BELOW_THRESHOLD:
9654	case VMX_EXIT_PREEMPT_TIMER:
9655	case VMX_EXIT_IO_INSTR:
9656	break;
9657
9658	/* Errors and unexpected events. */
9659	case VMX_EXIT_INIT_SIGNAL:
9660	case VMX_EXIT_SIPI:
9661	case VMX_EXIT_IO_SMI:
9662	case VMX_EXIT_SMI:
9663	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9664	case VMX_EXIT_ERR_MSR_LOAD:
9665	case VMX_EXIT_ERR_MACHINE_CHECK:
9666	break;
9667
9668	default:
9669	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9670	break;
9671	}
9672	#undef SET_BOTH
9673	#undef SET_EXIT
9674
9675	/*
9676	* Dtrace tracepoints go first. We do them here at once so we don't
9677	* have to copy the guest state saving and stuff a few dozen times.
9678	* Down side is that we've got to repeat the switch, though this time
9679	* we use enmEvent since the probes are a subset of what DBGF does.
9680	*/
9681	if (fDtrace1 \|\| fDtrace2)
9682	{
9683	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9684	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9685	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9686	switch (enmEvent1)
9687	{
9688	/** @todo consider which extra parameters would be helpful for each probe. */
9689	case DBGFEVENT_END: break;
9690	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
9691	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
9692	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
9693	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
9694	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
9695	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
9696	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
9697	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
9698	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
9699	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
9700	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
9701	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
9702	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
9703	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
9704	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
9705	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
9706	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
9707	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
9708	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9709	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9710	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
9711	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
9712	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
9713	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
9714	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
9715	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
9716	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
9717	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9718	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9719	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9720	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9721	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9722	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
9723	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9724	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
9725	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
9726	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
9727	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
9728	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
9729	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
9730	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
9731	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
9732	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
9733	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
9734	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
9735	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
9736	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
9737	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
9738	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
9739	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
9740	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
9741	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
9742	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
9743	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
9744	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
9745	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
9746	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
9747	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
9748	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
9749	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
9750	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
9751	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
9752	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
9753	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
9754	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
9755	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
9756	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9757	}
9758	switch (enmEvent2)
9759	{
9760	/** @todo consider which extra parameters would be helpful for each probe. */
9761	case DBGFEVENT_END: break;
9762	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
9763	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9764	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
9765	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
9766	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
9767	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
9768	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
9769	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
9770	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
9771	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9772	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9773	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9774	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9775	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9776	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
9777	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9778	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
9779	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
9780	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
9781	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
9782	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
9783	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
9784	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
9785	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
9786	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
9787	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
9788	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
9789	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
9790	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
9791	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
9792	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
9793	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
9794	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
9795	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
9796	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
9797	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
9798	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
9799	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
9800	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
9801	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
9802	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
9803	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
9804	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
9805	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
9806	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
9807	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
9808	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
9809	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
9810	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
9811	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
9812	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
9813	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
9814	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9815	}
9816	}
9817
9818	/*
9819	* Fire of the DBGF event, if enabled (our check here is just a quick one,
9820	* the DBGF call will do a full check).
9821	*
9822	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9823	* Note! If we have to events, we prioritize the first, i.e. the instruction
9824	* one, in order to avoid event nesting.
9825	*/
9826	PVM pVM = pVCpu->CTX_SUFF(pVM);
9827	if ( enmEvent1 != DBGFEVENT_END
9828	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9829	{
9830	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
9831	if (rcStrict != VINF_SUCCESS)
9832	return rcStrict;
9833	}
9834	else if ( enmEvent2 != DBGFEVENT_END
9835	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9836	{
9837	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
9838	if (rcStrict != VINF_SUCCESS)
9839	return rcStrict;
9840	}
9841
9842	return VINF_SUCCESS;
9843	}
9844
9845
9846	/**
9847	* Single-stepping VM-exit filtering.
9848	*
9849	* This is preprocessing the VM-exits and deciding whether we've gotten far
9850	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9851	* handling is performed.
9852	*
9853	* @returns Strict VBox status code (i.e. informational status codes too).
9854	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
9855	* @param pVmxTransient Pointer to the VMX-transient structure.
9856	* @param pDbgState The debug state.
9857	*/
9858	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
9859	{
9860	/*
9861	* Expensive (saves context) generic dtrace VM-exit probe.
9862	*/
9863	uint32_t const uExitReason = pVmxTransient->uExitReason;
9864	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9865	{ /* more likely */ }
9866	else
9867	{
9868	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9869	int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9870	AssertRC(rc);
9871	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
9872	}
9873
9874	/*
9875	* Check for host NMI, just to get that out of the way.
9876	*/
9877	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9878	{ /* normally likely */ }
9879	else
9880	{
9881	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9882	AssertRCReturn(rc2, rc2);
9883	uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
9884	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
9885	return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
9886	}
9887
9888	/*
9889	* Check for single stepping event if we're stepping.
9890	*/
9891	if (pVCpu->hm.s.fSingleInstruction)
9892	{
9893	switch (uExitReason)
9894	{
9895	case VMX_EXIT_MTF:
9896	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9897
9898	/* Various events: */
9899	case VMX_EXIT_XCPT_OR_NMI:
9900	case VMX_EXIT_EXT_INT:
9901	case VMX_EXIT_TRIPLE_FAULT:
9902	case VMX_EXIT_INT_WINDOW:
9903	case VMX_EXIT_NMI_WINDOW:
9904	case VMX_EXIT_TASK_SWITCH:
9905	case VMX_EXIT_TPR_BELOW_THRESHOLD:
9906	case VMX_EXIT_APIC_ACCESS:
9907	case VMX_EXIT_EPT_VIOLATION:
9908	case VMX_EXIT_EPT_MISCONFIG:
9909	case VMX_EXIT_PREEMPT_TIMER:
9910
9911	/* Instruction specific VM-exits: */
9912	case VMX_EXIT_CPUID:
9913	case VMX_EXIT_GETSEC:
9914	case VMX_EXIT_HLT:
9915	case VMX_EXIT_INVD:
9916	case VMX_EXIT_INVLPG:
9917	case VMX_EXIT_RDPMC:
9918	case VMX_EXIT_RDTSC:
9919	case VMX_EXIT_RSM:
9920	case VMX_EXIT_VMCALL:
9921	case VMX_EXIT_VMCLEAR:
9922	case VMX_EXIT_VMLAUNCH:
9923	case VMX_EXIT_VMPTRLD:
9924	case VMX_EXIT_VMPTRST:
9925	case VMX_EXIT_VMREAD:
9926	case VMX_EXIT_VMRESUME:
9927	case VMX_EXIT_VMWRITE:
9928	case VMX_EXIT_VMXOFF:
9929	case VMX_EXIT_VMXON:
9930	case VMX_EXIT_MOV_CRX:
9931	case VMX_EXIT_MOV_DRX:
9932	case VMX_EXIT_IO_INSTR:
9933	case VMX_EXIT_RDMSR:
9934	case VMX_EXIT_WRMSR:
9935	case VMX_EXIT_MWAIT:
9936	case VMX_EXIT_MONITOR:
9937	case VMX_EXIT_PAUSE:
9938	case VMX_EXIT_GDTR_IDTR_ACCESS:
9939	case VMX_EXIT_LDTR_TR_ACCESS:
9940	case VMX_EXIT_INVEPT:
9941	case VMX_EXIT_RDTSCP:
9942	case VMX_EXIT_INVVPID:
9943	case VMX_EXIT_WBINVD:
9944	case VMX_EXIT_XSETBV:
9945	case VMX_EXIT_RDRAND:
9946	case VMX_EXIT_INVPCID:
9947	case VMX_EXIT_VMFUNC:
9948	case VMX_EXIT_RDSEED:
9949	case VMX_EXIT_XSAVES:
9950	case VMX_EXIT_XRSTORS:
9951	{
9952	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
9953	AssertRCReturn(rc, rc);
9954	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
9955	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
9956	return VINF_EM_DBG_STEPPED;
9957	break;
9958	}
9959
9960	/* Errors and unexpected events: */
9961	case VMX_EXIT_INIT_SIGNAL:
9962	case VMX_EXIT_SIPI:
9963	case VMX_EXIT_IO_SMI:
9964	case VMX_EXIT_SMI:
9965	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9966	case VMX_EXIT_ERR_MSR_LOAD:
9967	case VMX_EXIT_ERR_MACHINE_CHECK:
9968	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9969	break;
9970
9971	default:
9972	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9973	break;
9974	}
9975	}
9976
9977	/*
9978	* Check for debugger event breakpoints and dtrace probes.
9979	*/
9980	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9981	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9982	{
9983	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
9984	if (rcStrict != VINF_SUCCESS)
9985	return rcStrict;
9986	}
9987
9988	/*
9989	* Normal processing.
9990	*/
9991	#ifdef HMVMX_USE_FUNCTION_TABLE
9992	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
9993	#else
9994	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
9995	#endif
9996	}
9997
9998
9999	/**
10000	* Single steps guest code using VT-x.
10001	*
10002	* @returns Strict VBox status code (i.e. informational status codes too).
10003	* @param pVCpu The cross context virtual CPU structure.
10004	*
10005	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10006	*/
10007	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu)
10008	{
10009	VMXTRANSIENT VmxTransient;
10010	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10011
10012	/* Set HMCPU indicators. */
10013	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10014	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
10015	pVCpu->hm.s.fDebugWantRdTscExit = false;
10016	pVCpu->hm.s.fUsingDebugLoop = true;
10017
10018	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10019	VMXRUNDBGSTATE DbgState;
10020	hmR0VmxRunDebugStateInit(pVCpu, &DbgState);
10021	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
10022
10023	/*
10024	* The loop.
10025	*/
10026	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10027	for (uint32_t cLoops = 0; ; cLoops++)
10028	{
10029	Assert(!HMR0SuspendPending());
10030	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10031	bool fStepping = pVCpu->hm.s.fSingleInstruction;
10032
10033	/*
10034	* Preparatory work for running guest code, this may force us to return
10035	* to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10036	*/
10037	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10038	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10039	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
10040	if (rcStrict != VINF_SUCCESS)
10041	break;
10042
10043	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10044	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10045
10046	/*
10047	* Now we can run the guest code.
10048	*/
10049	int rcRun = hmR0VmxRunGuest(pVCpu);
10050
10051	/*
10052	* Restore any residual host-state and save any bits shared between host
10053	* and guest into the guest-CPU state. Re-enables interrupts!
10054	*/
10055	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10056
10057	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10058	if (RT_SUCCESS(rcRun))
10059	{ /* very likely */ }
10060	else
10061	{
10062	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10063	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10064	return rcRun;
10065	}
10066
10067	/* Profile the VM-exit. */
10068	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10069	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10070	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10071	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10072	HMVMX_START_EXIT_DISPATCH_PROF();
10073
10074	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10075
10076	/*
10077	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10078	*/
10079	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
10080	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10081	if (rcStrict != VINF_SUCCESS)
10082	break;
10083	if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
10084	{
10085	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10086	rcStrict = VINF_EM_RAW_INTERRUPT;
10087	break;
10088	}
10089
10090	/*
10091	* Stepping: Did the RIP change, if so, consider it a single step.
10092	* Otherwise, make sure one of the TFs gets set.
10093	*/
10094	if (fStepping)
10095	{
10096	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
10097	AssertRC(rc);
10098	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
10099	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
10100	{
10101	rcStrict = VINF_EM_DBG_STEPPED;
10102	break;
10103	}
10104	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
10105	}
10106
10107	/*
10108	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
10109	*/
10110	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10111	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
10112	}
10113
10114	/*
10115	* Clear the X86_EFL_TF if necessary.
10116	*/
10117	if (pVCpu->hm.s.fClearTrapFlag)
10118	{
10119	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
10120	AssertRC(rc);
10121	pVCpu->hm.s.fClearTrapFlag = false;
10122	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
10123	}
10124	/** @todo there seems to be issues with the resume flag when the monitor trap
10125	* flag is pending without being used. Seen early in bios init when
10126	* accessing APIC page in protected mode. */
10127
10128	/*
10129	* Restore VM-exit control settings as we may not reenter this function the
10130	* next time around.
10131	*/
10132	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10133
10134	/* Restore HMCPU indicators. */
10135	pVCpu->hm.s.fUsingDebugLoop = false;
10136	pVCpu->hm.s.fDebugWantRdTscExit = false;
10137	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10138
10139	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10140	return rcStrict;
10141	}
10142
10143
10144	/** @} */
10145
10146
10147	/**
10148	* Checks if any expensive dtrace probes are enabled and we should go to the
10149	* debug loop.
10150	*
10151	* @returns true if we should use debug loop, false if not.
10152	*/
10153	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10154	{
10155	/* It's probably faster to OR the raw 32-bit counter variables together.
10156	Since the variables are in an array and the probes are next to one
10157	another (more or less), we have good locality. So, better read
10158	eight-nine cache lines ever time and only have one conditional, than
10159	128+ conditionals, right? */
10160	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10161	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
10162	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
10163	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
10164	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
10165	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
10166	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
10167	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
10168	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
10169	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
10170	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
10171	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
10172	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
10173	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
10174	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
10175	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
10176	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
10177	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
10178	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
10179	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10180	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10181	) != 0
10182	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10183	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10184	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10185	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10186	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
10187	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10188	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10189	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10190	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10191	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10192	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10193	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10194	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10195	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10196	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10197	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10198	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10199	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10200	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10201	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10202	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10203	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10204	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10205	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10206	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
10207	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
10208	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10209	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
10210	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10211	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10212	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10213	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10214	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10215	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10216	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10217	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10218	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10219	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10220	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10221	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10222	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10223	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10224	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10225	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10226	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10227	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10228	) != 0
10229	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10230	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
10231	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10232	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10233	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10234	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
10235	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10236	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10237	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10238	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10239	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10240	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10241	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10242	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10243	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10244	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10245	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10246	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10247	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10248	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10249	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10250	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10251	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10252	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10253	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10254	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
10255	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
10256	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10257	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
10258	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10259	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10260	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10261	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10262	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10263	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10264	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10265	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10266	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10267	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10268	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10269	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10270	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10271	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10272	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10273	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10274	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10275	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10276	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10277	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10278	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10279	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10280	) != 0;
10281	}
10282
10283
10284	/**
10285	* Runs the guest code using VT-x.
10286	*
10287	* @returns Strict VBox status code (i.e. informational status codes too).
10288	* @param pVCpu The cross context virtual CPU structure.
10289	*/
10290	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
10291	{
10292	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10293	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10294	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10295	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
10296
10297	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10298
10299	VBOXSTRICTRC rcStrict;
10300	if ( !pVCpu->hm.s.fUseDebugLoop
10301	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
10302	&& !DBGFIsStepping(pVCpu)
10303	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10304	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu);
10305	else
10306	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu);
10307
10308	if (rcStrict == VERR_EM_INTERPRETER)
10309	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10310	else if (rcStrict == VINF_EM_RESET)
10311	rcStrict = VINF_EM_TRIPLE_FAULT;
10312
10313	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
10314	if (RT_FAILURE(rc2))
10315	{
10316	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10317	rcStrict = rc2;
10318	}
10319	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10320	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10321	return rcStrict;
10322	}
10323
10324
10325	#ifndef HMVMX_USE_FUNCTION_TABLE
10326	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10327	{
10328	#ifdef DEBUG_ramshankar
10329	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10330	do { \
10331	if (a_fSave != 0) \
10332	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10333	VBOXSTRICTRC rcStrict = a_CallExpr; \
10334	if (a_fSave != 0) \
10335	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10336	return rcStrict; \
10337	} while (0)
10338	#else
10339	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10340	#endif
10341	switch (rcReason)
10342	{
10343	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
10344	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
10345	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
10346	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
10347	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
10348	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
10349	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
10350	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
10351	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
10352	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
10353	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
10354	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
10355	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
10356	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
10357	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
10358	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
10359	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
10360	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
10361	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
10362	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
10363	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
10364	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
10365	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
10366	case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
10367	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
10368	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
10369	case VMX_EXIT_GDTR_IDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10370	case VMX_EXIT_LDTR_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10371	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
10372	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
10373	case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
10374	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
10375	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
10376	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
10377
10378	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
10379	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
10380	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
10381	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
10382	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
10383	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
10384	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
10385	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
10386	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
10387
10388	case VMX_EXIT_VMCLEAR:
10389	case VMX_EXIT_VMLAUNCH:
10390	case VMX_EXIT_VMPTRLD:
10391	case VMX_EXIT_VMPTRST:
10392	case VMX_EXIT_VMREAD:
10393	case VMX_EXIT_VMRESUME:
10394	case VMX_EXIT_VMWRITE:
10395	case VMX_EXIT_VMXOFF:
10396	case VMX_EXIT_VMXON:
10397	case VMX_EXIT_INVEPT:
10398	case VMX_EXIT_INVVPID:
10399	case VMX_EXIT_VMFUNC:
10400	case VMX_EXIT_XSAVES:
10401	case VMX_EXIT_XRSTORS:
10402	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
10403
10404	case VMX_EXIT_ENCLS:
10405	case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10406	case VMX_EXIT_PML_FULL:
10407	default:
10408	return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
10409	}
10410	#undef VMEXIT_CALL_RET
10411	}
10412	#endif /* !HMVMX_USE_FUNCTION_TABLE */
10413
10414
10415	#ifdef VBOX_STRICT
10416	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10417	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10418	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10419
10420	# define HMVMX_ASSERT_PREEMPT_CPUID() \
10421	do { \
10422	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10423	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10424	} while (0)
10425
10426	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10427	do { \
10428	AssertPtr((a_pVCpu)); \
10429	AssertPtr((a_pVmxTransient)); \
10430	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
10431	Assert(ASMIntAreEnabled()); \
10432	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10433	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10434	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
10435	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10436	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
10437	HMVMX_ASSERT_PREEMPT_CPUID(); \
10438	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10439	} while (0)
10440
10441	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10442	do { \
10443	Log4Func(("\n")); \
10444	} while (0)
10445	#else
10446	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10447	do { \
10448	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10449	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
10450	} while (0)
10451	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
10452	#endif
10453
10454
10455	/**
10456	* Advances the guest RIP by the specified number of bytes.
10457	*
10458	* @param pVCpu The cross context virtual CPU structure.
10459	* @param cbInstr Number of bytes to advance the RIP by.
10460	*
10461	* @remarks No-long-jump zone!!!
10462	*/
10463	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
10464	{
10465	/* Advance the RIP. */
10466	pVCpu->cpum.GstCtx.rip += cbInstr;
10467	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10468
10469	/* Update interrupt inhibition. */
10470	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10471	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
10472	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10473	}
10474
10475
10476	/**
10477	* Advances the guest RIP after reading it from the VMCS.
10478	*
10479	* @returns VBox status code, no informational status codes.
10480	* @param pVCpu The cross context virtual CPU structure.
10481	* @param pVmxTransient Pointer to the VMX transient structure.
10482	*
10483	* @remarks No-long-jump zone!!!
10484	*/
10485	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10486	{
10487	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10488	rc \|= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
10489	AssertRCReturn(rc, rc);
10490
10491	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
10492
10493	/*
10494	* Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10495	* pending debug exception field as it takes care of priority of events.
10496	*
10497	* See Intel spec. 32.2.1 "Debug Exceptions".
10498	*/
10499	if ( !pVCpu->hm.s.fSingleInstruction
10500	&& pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
10501	{
10502	rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10503	AssertRCReturn(rc, rc);
10504	}
10505
10506	return VINF_SUCCESS;
10507	}
10508
10509
10510	/**
10511	* Tries to determine what part of the guest-state VT-x has deemed as invalid
10512	* and update error record fields accordingly.
10513	*
10514	* @return VMX_IGS_* return codes.
10515	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10516	* wrong with the guest state.
10517	*
10518	* @param pVCpu The cross context virtual CPU structure.
10519	*
10520	* @remarks This function assumes our cache of the VMCS controls
10521	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10522	*/
10523	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu)
10524	{
10525	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10526	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10527	uError = (err); \
10528	break; \
10529	} else do { } while (0)
10530
10531	int rc;
10532	PVM pVM = pVCpu->CTX_SUFF(pVM);
10533	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10534	uint32_t uError = VMX_IGS_ERROR;
10535	uint32_t u32Val;
10536	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10537
10538	do
10539	{
10540	/*
10541	* CR0.
10542	*/
10543	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10544	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10545	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10546	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10547	if (fUnrestrictedGuest)
10548	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
10549
10550	uint32_t u32GuestCr0;
10551	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10552	AssertRCBreak(rc);
10553	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10554	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10555	if ( !fUnrestrictedGuest
10556	&& (u32GuestCr0 & X86_CR0_PG)
10557	&& !(u32GuestCr0 & X86_CR0_PE))
10558	{
10559	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10560	}
10561
10562	/*
10563	* CR4.
10564	*/
10565	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10566	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10567
10568	uint32_t u32GuestCr4;
10569	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10570	AssertRCBreak(rc);
10571	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10572	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10573
10574	/*
10575	* IA32_DEBUGCTL MSR.
10576	*/
10577	uint64_t u64Val;
10578	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10579	AssertRCBreak(rc);
10580	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10581	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10582	{
10583	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10584	}
10585	uint64_t u64DebugCtlMsr = u64Val;
10586
10587	#ifdef VBOX_STRICT
10588	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10589	AssertRCBreak(rc);
10590	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10591	#endif
10592	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
10593
10594	/*
10595	* RIP and RFLAGS.
10596	*/
10597	uint32_t u32Eflags;
10598	#if HC_ARCH_BITS == 64
10599	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10600	AssertRCBreak(rc);
10601	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10602	if ( !fLongModeGuest
10603	\|\| !pCtx->cs.Attr.n.u1Long)
10604	{
10605	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10606	}
10607	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10608	* must be identical if the "IA-32e mode guest" VM-entry
10609	* control is 1 and CS.L is 1. No check applies if the
10610	* CPU supports 64 linear-address bits. */
10611
10612	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10613	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10614	AssertRCBreak(rc);
10615	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10616	VMX_IGS_RFLAGS_RESERVED);
10617	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10618	u32Eflags = u64Val;
10619	#else
10620	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10621	AssertRCBreak(rc);
10622	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10623	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10624	#endif
10625
10626	if ( fLongModeGuest
10627	\|\| ( fUnrestrictedGuest
10628	&& !(u32GuestCr0 & X86_CR0_PE)))
10629	{
10630	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10631	}
10632
10633	uint32_t u32EntryInfo;
10634	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10635	AssertRCBreak(rc);
10636	if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10637	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10638	{
10639	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10640	}
10641
10642	/*
10643	* 64-bit checks.
10644	*/
10645	#if HC_ARCH_BITS == 64
10646	if (fLongModeGuest)
10647	{
10648	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10649	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10650	}
10651
10652	if ( !fLongModeGuest
10653	&& (u32GuestCr4 & X86_CR4_PCIDE))
10654	{
10655	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10656	}
10657
10658	/** @todo CR3 field must be such that bits 63:52 and bits in the range
10659	* 51:32 beyond the processor's physical-address width are 0. */
10660
10661	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10662	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10663	{
10664	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10665	}
10666
10667	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10668	AssertRCBreak(rc);
10669	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10670
10671	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10672	AssertRCBreak(rc);
10673	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10674	#endif
10675
10676	/*
10677	* PERF_GLOBAL MSR.
10678	*/
10679	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
10680	{
10681	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10682	AssertRCBreak(rc);
10683	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10684	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10685	}
10686
10687	/*
10688	* PAT MSR.
10689	*/
10690	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
10691	{
10692	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10693	AssertRCBreak(rc);
10694	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10695	for (unsigned i = 0; i < 8; i++)
10696	{
10697	uint8_t u8Val = (u64Val & 0xff);
10698	if ( u8Val != 0 /* UC */
10699	&& u8Val != 1 /* WC */
10700	&& u8Val != 4 /* WT */
10701	&& u8Val != 5 /* WP */
10702	&& u8Val != 6 /* WB */
10703	&& u8Val != 7 /* UC- */)
10704	{
10705	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10706	}
10707	u64Val >>= 8;
10708	}
10709	}
10710
10711	/*
10712	* EFER MSR.
10713	*/
10714	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
10715	{
10716	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10717	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10718	AssertRCBreak(rc);
10719	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10720	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10721	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10722	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
10723	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10724	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
10725	* iemVmxVmentryCheckGuestState(). */
10726	HMVMX_CHECK_BREAK( fUnrestrictedGuest
10727	\|\| !(u32GuestCr0 & X86_CR0_PG)
10728	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10729	VMX_IGS_EFER_LMA_LME_MISMATCH);
10730	}
10731
10732	/*
10733	* Segment registers.
10734	*/
10735	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10736	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10737	if (!(u32Eflags & X86_EFL_VM))
10738	{
10739	/* CS */
10740	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10741	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10742	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10743	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10744	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10745	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10746	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10747	/* CS cannot be loaded with NULL in protected mode. */
10748	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10749	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10750	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
10751	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10752	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
10753	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10754	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10755	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10756	else
10757	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10758
10759	/* SS */
10760	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10761	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10762	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10763	if ( !(pCtx->cr0 & X86_CR0_PE)
10764	\|\| pCtx->cs.Attr.n.u4Type == 3)
10765	{
10766	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10767	}
10768	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10769	{
10770	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10771	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10772	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10773	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10774	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10775	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10776	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10777	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10778	}
10779
10780	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmenReg(). */
10781	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10782	{
10783	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10784	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10785	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10786	\|\| pCtx->ds.Attr.n.u4Type > 11
10787	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10788	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10789	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10790	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10791	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10792	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10793	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10794	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10795	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10796	}
10797	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10798	{
10799	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10800	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10801	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10802	\|\| pCtx->es.Attr.n.u4Type > 11
10803	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10804	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10805	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10806	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10807	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10808	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10809	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10810	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10811	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10812	}
10813	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10814	{
10815	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10816	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10817	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10818	\|\| pCtx->fs.Attr.n.u4Type > 11
10819	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10820	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10821	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10822	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10823	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10824	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10825	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10826	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10827	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10828	}
10829	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10830	{
10831	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10832	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10833	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10834	\|\| pCtx->gs.Attr.n.u4Type > 11
10835	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10836	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10837	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10838	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10839	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10840	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10841	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10842	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10843	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10844	}
10845	/* 64-bit capable CPUs. */
10846	#if HC_ARCH_BITS == 64
10847	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10848	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10849	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10850	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10851	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10852	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
10853	VMX_IGS_LONGMODE_SS_BASE_INVALID);
10854	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
10855	VMX_IGS_LONGMODE_DS_BASE_INVALID);
10856	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
10857	VMX_IGS_LONGMODE_ES_BASE_INVALID);
10858	#endif
10859	}
10860	else
10861	{
10862	/* V86 mode checks. */
10863	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10864	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10865	{
10866	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10867	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10868	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10869	}
10870	else
10871	{
10872	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10873	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10874	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10875	}
10876
10877	/* CS */
10878	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10879	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10880	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10881	/* SS */
10882	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10883	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10884	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10885	/* DS */
10886	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10887	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10888	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10889	/* ES */
10890	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10891	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10892	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10893	/* FS */
10894	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10895	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10896	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10897	/* GS */
10898	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10899	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10900	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10901	/* 64-bit capable CPUs. */
10902	#if HC_ARCH_BITS == 64
10903	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10904	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10905	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10906	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10907	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10908	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
10909	VMX_IGS_LONGMODE_SS_BASE_INVALID);
10910	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
10911	VMX_IGS_LONGMODE_DS_BASE_INVALID);
10912	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
10913	VMX_IGS_LONGMODE_ES_BASE_INVALID);
10914	#endif
10915	}
10916
10917	/*
10918	* TR.
10919	*/
10920	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10921	/* 64-bit capable CPUs. */
10922	#if HC_ARCH_BITS == 64
10923	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10924	#endif
10925	if (fLongModeGuest)
10926	{
10927	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10928	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10929	}
10930	else
10931	{
10932	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10933	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10934	VMX_IGS_TR_ATTR_TYPE_INVALID);
10935	}
10936	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10937	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10938	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10939	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10940	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10941	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10942	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10943	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10944
10945	/*
10946	* GDTR and IDTR.
10947	*/
10948	#if HC_ARCH_BITS == 64
10949	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10950	AssertRCBreak(rc);
10951	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10952
10953	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10954	AssertRCBreak(rc);
10955	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10956	#endif
10957
10958	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10959	AssertRCBreak(rc);
10960	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10961
10962	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10963	AssertRCBreak(rc);
10964	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10965
10966	/*
10967	* Guest Non-Register State.
10968	*/
10969	/* Activity State. */
10970	uint32_t u32ActivityState;
10971	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10972	AssertRCBreak(rc);
10973	HMVMX_CHECK_BREAK( !u32ActivityState
10974	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
10975	VMX_IGS_ACTIVITY_STATE_INVALID);
10976	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10977	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10978	uint32_t u32IntrState;
10979	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
10980	AssertRCBreak(rc);
10981	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
10982	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10983	{
10984	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10985	}
10986
10987	/** @todo Activity state and injecting interrupts. Left as a todo since we
10988	* currently don't use activity states but ACTIVE. */
10989
10990	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10991	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10992
10993	/* Guest interruptibility-state. */
10994	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10995	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
10996	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10997	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10998	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10999	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
11000	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11001	if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
11002	{
11003	if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
11004	{
11005	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
11006	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
11007	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11008	}
11009	else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
11010	{
11011	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
11012	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11013	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
11014	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11015	}
11016	}
11017	/** @todo Assumes the processor is not in SMM. */
11018	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
11019	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11020	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
11021	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
11022	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11023	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
11024	&& VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
11025	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
11026	{
11027	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
11028	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11029	}
11030
11031	/* Pending debug exceptions. */
11032	#if HC_ARCH_BITS == 64
11033	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
11034	AssertRCBreak(rc);
11035	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11036	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11037	u32Val = u64Val; /* For pending debug exceptions checks below. */
11038	#else
11039	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
11040	AssertRCBreak(rc);
11041	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11042	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11043	#endif
11044
11045	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
11046	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
11047	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11048	{
11049	if ( (u32Eflags & X86_EFL_TF)
11050	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11051	{
11052	/* Bit 14 is PendingDebug.BS. */
11053	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11054	}
11055	if ( !(u32Eflags & X86_EFL_TF)
11056	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11057	{
11058	/* Bit 14 is PendingDebug.BS. */
11059	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11060	}
11061	}
11062
11063	/* VMCS link pointer. */
11064	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11065	AssertRCBreak(rc);
11066	if (u64Val != UINT64_C(0xffffffffffffffff))
11067	{
11068	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11069	/** @todo Bits beyond the processor's physical-address width MBZ. */
11070	/** @todo 32-bit located in memory referenced by value of this field (as a
11071	* physical address) must contain the processor's VMCS revision ID. */
11072	/** @todo SMM checks. */
11073	}
11074
11075	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11076	* not using Nested Paging? */
11077	if ( pVM->hm.s.fNestedPaging
11078	&& !fLongModeGuest
11079	&& CPUMIsGuestInPAEModeEx(pCtx))
11080	{
11081	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11082	AssertRCBreak(rc);
11083	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11084
11085	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11086	AssertRCBreak(rc);
11087	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11088
11089	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11090	AssertRCBreak(rc);
11091	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11092
11093	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11094	AssertRCBreak(rc);
11095	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11096	}
11097
11098	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11099	if (uError == VMX_IGS_ERROR)
11100	uError = VMX_IGS_REASON_NOT_FOUND;
11101	} while (0);
11102
11103	pVCpu->hm.s.u32HMError = uError;
11104	return uError;
11105
11106	#undef HMVMX_ERROR_BREAK
11107	#undef HMVMX_CHECK_BREAK
11108	}
11109
11110
11111	/** @name VM-exit handlers.
11112	* @{
11113	*/
11114	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11115	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11116	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11117
11118	/**
11119	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11120	*/
11121	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11122	{
11123	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11124	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11125	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11126	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11127	return VINF_SUCCESS;
11128	return VINF_EM_RAW_INTERRUPT;
11129	}
11130
11131
11132	/**
11133	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11134	*/
11135	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11136	{
11137	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11138	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11139
11140	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11141	AssertRCReturn(rc, rc);
11142
11143	uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11144	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
11145	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
11146	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11147
11148	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11149	{
11150	/*
11151	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
11152	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
11153	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
11154	*
11155	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11156	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
11157	*/
11158	VMXDispatchHostNmi();
11159	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11160	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11161	return VINF_SUCCESS;
11162	}
11163
11164	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11165	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
11166	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11167	{ /* likely */ }
11168	else
11169	{
11170	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11171	rcStrictRc1 = VINF_SUCCESS;
11172	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11173	return rcStrictRc1;
11174	}
11175
11176	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11177	uint32_t uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
11178	switch (uIntType)
11179	{
11180	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11181	Assert(uVector == X86_XCPT_DB);
11182	RT_FALL_THRU();
11183	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11184	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
11185	RT_FALL_THRU();
11186	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11187	{
11188	/*
11189	* If there's any exception caused as a result of event injection, the resulting
11190	* secondary/final execption will be pending, we shall continue guest execution
11191	* after injecting the event. The page-fault case is complicated and we manually
11192	* handle any currently pending event in hmR0VmxExitXcptPF.
11193	*/
11194	if (!pVCpu->hm.s.Event.fPending)
11195	{ /* likely */ }
11196	else if (uVector != X86_XCPT_PF)
11197	{
11198	rc = VINF_SUCCESS;
11199	break;
11200	}
11201
11202	switch (uVector)
11203	{
11204	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
11205	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
11206	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
11207	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
11208	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
11209	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
11210
11211	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11212	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11213	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11214	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11215	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11216	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11217	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11218	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11219	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11220	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11221	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11222	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11223	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11224	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11225	default:
11226	{
11227	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11228	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11229	{
11230	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11231	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11232	Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
11233
11234	rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0);
11235	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11236	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11237	AssertRCReturn(rc, rc);
11238	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11239	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11240	0 /* GCPtrFaultAddress */);
11241	}
11242	else
11243	{
11244	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11245	pVCpu->hm.s.u32HMError = uVector;
11246	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11247	}
11248	break;
11249	}
11250	}
11251	break;
11252	}
11253
11254	default:
11255	{
11256	pVCpu->hm.s.u32HMError = uExitIntInfo;
11257	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11258	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
11259	break;
11260	}
11261	}
11262	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11263	return rc;
11264	}
11265
11266
11267	/**
11268	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11269	*/
11270	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11271	{
11272	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11273
11274	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11275	hmR0VmxClearIntWindowExitVmcs(pVCpu);
11276
11277	/* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11278	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11279	return VINF_SUCCESS;
11280	}
11281
11282
11283	/**
11284	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11285	*/
11286	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11287	{
11288	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11289	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)))
11290	{
11291	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11292	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11293	}
11294
11295	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11296
11297	/*
11298	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11299	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11300	*/
11301	uint32_t fIntrState = 0;
11302	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11303	AssertRCReturn(rc, rc);
11304
11305	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
11306	if ( fBlockSti
11307	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11308	{
11309	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11310	}
11311
11312	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11313	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11314
11315	/* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11316	return VINF_SUCCESS;
11317	}
11318
11319
11320	/**
11321	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11322	*/
11323	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11324	{
11325	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11326	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11327	}
11328
11329
11330	/**
11331	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11332	*/
11333	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11334	{
11335	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11336	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11337	}
11338
11339
11340	/**
11341	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11342	*/
11343	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11344	{
11345	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11346
11347	/*
11348	* Get the state we need and update the exit history entry.
11349	*/
11350	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11351	rc \|= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
11352	AssertRCReturn(rc, rc);
11353
11354	VBOXSTRICTRC rcStrict;
11355	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11356	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
11357	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11358	if (!pExitRec)
11359	{
11360	/*
11361	* Regular CPUID instruction execution.
11362	*/
11363	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
11364	if (rcStrict == VINF_SUCCESS)
11365	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11366	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11367	{
11368	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11369	rcStrict = VINF_SUCCESS;
11370	}
11371	}
11372	else
11373	{
11374	/*
11375	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
11376	*/
11377	int rc2 = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11378	AssertRCReturn(rc2, rc2);
11379
11380	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11381	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11382
11383	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11384	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11385
11386	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11387	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11388	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11389	}
11390	return rcStrict;
11391	}
11392
11393
11394	/**
11395	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11396	*/
11397	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11398	{
11399	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11400	int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR4);
11401	AssertRCReturn(rc, rc);
11402
11403	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
11404	return VINF_EM_RAW_EMULATE_INSTR;
11405
11406	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11407	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11408	}
11409
11410
11411	/**
11412	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11413	*/
11414	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11415	{
11416	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11417	int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11418	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11419	AssertRCReturn(rc, rc);
11420
11421	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11422	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11423	{
11424	/* If we get a spurious VM-exit when offsetting is enabled,
11425	we must reset offsetting on VM-reentry. See @bugref{6634}. */
11426	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11427	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11428	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11429	}
11430	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11431	{
11432	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11433	rcStrict = VINF_SUCCESS;
11434	}
11435	return rcStrict;
11436	}
11437
11438
11439	/**
11440	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11441	*/
11442	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11443	{
11444	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11445	int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
11446	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11447	AssertRCReturn(rc, rc);
11448
11449	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11450	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11451	{
11452	/* If we get a spurious VM-exit when offsetting is enabled,
11453	we must reset offsetting on VM-reentry. See @bugref{6634}. */
11454	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11455	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11456	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11457	}
11458	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11459	{
11460	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11461	rcStrict = VINF_SUCCESS;
11462	}
11463	return rcStrict;
11464	}
11465
11466
11467	/**
11468	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11469	*/
11470	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11471	{
11472	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11473	int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
11474	AssertRCReturn(rc, rc);
11475
11476	PVM pVM = pVCpu->CTX_SUFF(pVM);
11477	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11478	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11479	if (RT_LIKELY(rc == VINF_SUCCESS))
11480	{
11481	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11482	Assert(pVmxTransient->cbInstr == 2);
11483	}
11484	else
11485	{
11486	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11487	rc = VERR_EM_INTERPRETER;
11488	}
11489	return rc;
11490	}
11491
11492
11493	/**
11494	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11495	*/
11496	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11497	{
11498	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11499
11500	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11501	if (EMAreHypercallInstructionsEnabled(pVCpu))
11502	{
11503	int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
11504	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
11505	AssertRCReturn(rc, rc);
11506
11507	/* Perform the hypercall. */
11508	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
11509	if (rcStrict == VINF_SUCCESS)
11510	{
11511	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11512	AssertRCReturn(rc, rc);
11513	}
11514	else
11515	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11516	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11517	\|\| RT_FAILURE(rcStrict));
11518
11519	/* If the hypercall changes anything other than guest's general-purpose registers,
11520	we would need to reload the guest changed bits here before VM-entry. */
11521	}
11522	else
11523	Log4Func(("Hypercalls not enabled\n"));
11524
11525	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11526	if (RT_FAILURE(rcStrict))
11527	{
11528	hmR0VmxSetPendingXcptUD(pVCpu);
11529	rcStrict = VINF_SUCCESS;
11530	}
11531
11532	return rcStrict;
11533	}
11534
11535
11536	/**
11537	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11538	*/
11539	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11540	{
11541	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11542	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
11543
11544	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11545	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11546	rc \|= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
11547	AssertRCReturn(rc, rc);
11548
11549	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
11550
11551	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
11552	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11553	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11554	{
11555	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11556	rcStrict = VINF_SUCCESS;
11557	}
11558	else
11559	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n", pVmxTransient->uExitQual,
11560	VBOXSTRICTRC_VAL(rcStrict)));
11561	return rcStrict;
11562	}
11563
11564
11565	/**
11566	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11567	*/
11568	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11569	{
11570	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11571	int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
11572	AssertRCReturn(rc, rc);
11573
11574	PVM pVM = pVCpu->CTX_SUFF(pVM);
11575	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11576	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11577	if (RT_LIKELY(rc == VINF_SUCCESS))
11578	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11579	else
11580	{
11581	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11582	rc = VERR_EM_INTERPRETER;
11583	}
11584	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11585	return rc;
11586	}
11587
11588
11589	/**
11590	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11591	*/
11592	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11593	{
11594	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11595	int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
11596	AssertRCReturn(rc, rc);
11597
11598	PVM pVM = pVCpu->CTX_SUFF(pVM);
11599	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11600	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11601	rc = VBOXSTRICTRC_VAL(rc2);
11602	if (RT_LIKELY( rc == VINF_SUCCESS
11603	\|\| rc == VINF_EM_HALT))
11604	{
11605	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11606	AssertRCReturn(rc3, rc3);
11607
11608	if ( rc == VINF_EM_HALT
11609	&& EMMonitorWaitShouldContinue(pVCpu, pCtx))
11610	rc = VINF_SUCCESS;
11611	}
11612	else
11613	{
11614	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11615	rc = VERR_EM_INTERPRETER;
11616	}
11617	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
11618	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11619	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11620	return rc;
11621	}
11622
11623
11624	/**
11625	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11626	*/
11627	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11628	{
11629	/*
11630	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11631	* mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11632	* treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11633	* VMX root operation. If we get here, something funny is going on.
11634	*
11635	* See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11636	*/
11637	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11638	AssertMsgFailed(("Unexpected RSM VM-exit\n"));
11639	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11640	}
11641
11642
11643	/**
11644	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11645	*/
11646	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11647	{
11648	/*
11649	* This can only happen if we support dual-monitor treatment of SMI, which can be activated
11650	* by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11651	* this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11652	* an SMI. If we get here, something funny is going on.
11653	*
11654	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11655	* See Intel spec. 25.3 "Other Causes of VM-Exits"
11656	*/
11657	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11658	AssertMsgFailed(("Unexpected SMI VM-exit\n"));
11659	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11660	}
11661
11662
11663	/**
11664	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11665	*/
11666	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11667	{
11668	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11669	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11670	AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
11671	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11672	}
11673
11674
11675	/**
11676	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11677	*/
11678	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11679	{
11680	/*
11681	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11682	* We don't make use of it as our guests don't have direct access to the host LAPIC.
11683	* See Intel spec. 25.3 "Other Causes of VM-exits".
11684	*/
11685	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11686	AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
11687	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11688	}
11689
11690
11691	/**
11692	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11693	* VM-exit.
11694	*/
11695	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11696	{
11697	/*
11698	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11699	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11700	*
11701	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11702	* See Intel spec. "23.8 Restrictions on VMX operation".
11703	*/
11704	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11705	return VINF_SUCCESS;
11706	}
11707
11708
11709	/**
11710	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11711	* VM-exit.
11712	*/
11713	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11714	{
11715	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11716	return VINF_EM_RESET;
11717	}
11718
11719
11720	/**
11721	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11722	*/
11723	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11724	{
11725	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11726	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_HLT_EXIT);
11727
11728	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11729	rc \|= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RFLAGS);
11730	AssertRCReturn(rc, rc);
11731
11732	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
11733	rc = VINF_SUCCESS;
11734	else
11735	rc = VINF_EM_HALT;
11736
11737	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11738	if (rc != VINF_SUCCESS)
11739	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11740	return rc;
11741	}
11742
11743
11744	/**
11745	* VM-exit handler for instructions that result in a \#UD exception delivered to
11746	* the guest.
11747	*/
11748	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11749	{
11750	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11751	hmR0VmxSetPendingXcptUD(pVCpu);
11752	return VINF_SUCCESS;
11753	}
11754
11755
11756	/**
11757	* VM-exit handler for expiry of the VMX preemption timer.
11758	*/
11759	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11760	{
11761	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11762
11763	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11764	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11765
11766	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11767	PVM pVM = pVCpu->CTX_SUFF(pVM);
11768	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11769	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11770	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11771	}
11772
11773
11774	/**
11775	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11776	*/
11777	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11778	{
11779	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11780
11781	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11782	rc \|= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
11783	AssertRCReturn(rc, rc);
11784
11785	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11786	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
11787	: HM_CHANGED_RAISED_XCPT_MASK);
11788
11789	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11790	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
11791
11792	return rcStrict;
11793	}
11794
11795
11796	/**
11797	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11798	*/
11799	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11800	{
11801	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11802	/** @todo Use VM-exit instruction information. */
11803	return VERR_EM_INTERPRETER;
11804	}
11805
11806
11807	/**
11808	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11809	* Error VM-exit.
11810	*/
11811	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11812	{
11813	int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11814	AssertRCReturn(rc, rc);
11815	rc = hmR0VmxCheckVmcsCtls(pVCpu);
11816	if (RT_FAILURE(rc))
11817	return rc;
11818
11819	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
11820	NOREF(uInvalidReason);
11821
11822	#ifdef VBOX_STRICT
11823	uint32_t fIntrState;
11824	RTHCUINTREG uHCReg;
11825	uint64_t u64Val;
11826	uint32_t u32Val;
11827
11828	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11829	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11830	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11831	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11832	AssertRCReturn(rc, rc);
11833
11834	Log4(("uInvalidReason %u\n", uInvalidReason));
11835	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11836	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11837	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11838	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
11839
11840	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11841	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11842	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11843	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11844	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11845	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11846	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11847	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11848	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11849	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11850	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11851	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11852
11853	hmR0DumpRegs(pVCpu);
11854	#else
11855	NOREF(pVmxTransient);
11856	#endif
11857
11858	return VERR_VMX_INVALID_GUEST_STATE;
11859	}
11860
11861
11862	/**
11863	* VM-exit handler for VM-entry failure due to an MSR-load
11864	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11865	*/
11866	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11867	{
11868	AssertMsgFailed(("Unexpected MSR-load exit\n"));
11869	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11870	}
11871
11872
11873	/**
11874	* VM-exit handler for VM-entry failure due to a machine-check event
11875	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11876	*/
11877	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11878	{
11879	AssertMsgFailed(("Unexpected machine-check event exit\n"));
11880	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11881	}
11882
11883
11884	/**
11885	* VM-exit handler for all undefined reasons. Should never ever happen.. in
11886	* theory.
11887	*/
11888	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11889	{
11890	RT_NOREF2(pVCpu, pVmxTransient);
11891	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
11892	return VERR_VMX_UNDEFINED_EXIT_CODE;
11893	}
11894
11895
11896	/**
11897	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11898	* (VMX_EXIT_GDTR_IDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11899	* Conditional VM-exit.
11900	*/
11901	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11902	{
11903	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11904
11905	/* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
11906	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11907	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
11908	return VERR_EM_INTERPRETER;
11909	AssertMsgFailed(("Unexpected XDTR access\n"));
11910	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11911	}
11912
11913
11914	/**
11915	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11916	*/
11917	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11918	{
11919	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11920
11921	/* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
11922	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
11923	return VERR_EM_INTERPRETER;
11924	AssertMsgFailed(("Unexpected RDRAND exit\n"));
11925	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11926	}
11927
11928
11929	/**
11930	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11931	*/
11932	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11933	{
11934	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11935
11936	/** @todo Optimize this: We currently drag in in the whole MSR state
11937	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11938	* MSRs required. That would require changes to IEM and possibly CPUM too.
11939	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11940	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; NOREF(idMsr); /* Save it. */
11941	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11942	rc \|= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
11943	AssertRCReturn(rc, rc);
11944
11945	Log4Func(("ecx=%#RX32\n", idMsr));
11946
11947	#ifdef VBOX_STRICT
11948	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
11949	{
11950	if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr)
11951	&& idMsr != MSR_K6_EFER)
11952	{
11953	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
11954	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11955	}
11956	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11957	{
11958	VMXMSREXITREAD enmRead;
11959	VMXMSREXITWRITE enmWrite;
11960	int rc2 = HMVmxGetMsrPermission(pVCpu->hm.s.vmx.pvMsrBitmap, idMsr, &enmRead, &enmWrite);
11961	AssertRCReturn(rc2, rc2);
11962	if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11963	{
11964	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
11965	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11966	}
11967	}
11968	}
11969	#endif
11970
11971	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
11972	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11973	if (rcStrict == VINF_SUCCESS)
11974	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
11975	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
11976	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11977	{
11978	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11979	rcStrict = VINF_SUCCESS;
11980	}
11981	else
11982	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11983
11984	return rcStrict;
11985	}
11986
11987
11988	/**
11989	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11990	*/
11991	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11992	{
11993	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11994
11995	/** @todo Optimize this: We currently drag in in the whole MSR state
11996	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11997	* MSRs required. That would require changes to IEM and possibly CPUM too.
11998	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11999	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; /* Save it. */
12000	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12001	rc \|= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
12002	AssertRCReturn(rc, rc);
12003
12004	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
12005
12006	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
12007	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12008
12009	if (rcStrict == VINF_SUCCESS)
12010	{
12011	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
12012
12013	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
12014	if ( idMsr == MSR_IA32_APICBASE
12015	\|\| ( idMsr >= MSR_IA32_X2APIC_START
12016	&& idMsr <= MSR_IA32_X2APIC_END))
12017	{
12018	/*
12019	* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12020	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before IEM changes it.
12021	*/
12022	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
12023	}
12024	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12025	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12026	else if (idMsr == MSR_K6_EFER)
12027	{
12028	/*
12029	* If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12030	* even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12031	* the other bits as well, SCE and NXE. See @bugref{7368}.
12032	*/
12033	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_CTLS
12034	\| HM_CHANGED_VMX_EXIT_CTLS);
12035	}
12036
12037	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12038	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
12039	{
12040	switch (idMsr)
12041	{
12042	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12043	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12044	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12045	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
12046	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
12047	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
12048	default:
12049	{
12050	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
12051	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12052	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
12053	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
12054	break;
12055	}
12056	}
12057	}
12058	#ifdef VBOX_STRICT
12059	else
12060	{
12061	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12062	switch (idMsr)
12063	{
12064	case MSR_IA32_SYSENTER_CS:
12065	case MSR_IA32_SYSENTER_EIP:
12066	case MSR_IA32_SYSENTER_ESP:
12067	case MSR_K8_FS_BASE:
12068	case MSR_K8_GS_BASE:
12069	{
12070	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
12071	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12072	}
12073
12074	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12075	default:
12076	{
12077	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
12078	{
12079	/* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
12080	if (idMsr != MSR_K6_EFER)
12081	{
12082	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12083	idMsr));
12084	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12085	}
12086	}
12087
12088	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
12089	{
12090	VMXMSREXITREAD enmRead;
12091	VMXMSREXITWRITE enmWrite;
12092	int rc2 = HMVmxGetMsrPermission(pVCpu->hm.s.vmx.pvMsrBitmap, idMsr, &enmRead, &enmWrite);
12093	AssertRCReturn(rc2, rc2);
12094	if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12095	{
12096	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
12097	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12098	}
12099	}
12100	break;
12101	}
12102	}
12103	}
12104	#endif /* VBOX_STRICT */
12105	}
12106	else if (rcStrict == VINF_IEM_RAISED_XCPT)
12107	{
12108	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12109	rcStrict = VINF_SUCCESS;
12110	}
12111	else
12112	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12113
12114	return rcStrict;
12115	}
12116
12117
12118	/**
12119	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12120	*/
12121	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12122	{
12123	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12124	/** @todo The guest has likely hit a contended spinlock. We might want to
12125	* poke a schedule different guest VCPU. */
12126	return VINF_EM_RAW_INTERRUPT;
12127	}
12128
12129
12130	/**
12131	* VM-exit handler for when the TPR value is lowered below the specified
12132	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12133	*/
12134	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12135	{
12136	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12137	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
12138
12139	/*
12140	* The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12141	* pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12142	*/
12143	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12144	return VINF_SUCCESS;
12145	}
12146
12147
12148	/**
12149	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12150	* VM-exit.
12151	*
12152	* @retval VINF_SUCCESS when guest execution can continue.
12153	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12154	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12155	* interpreter.
12156	*/
12157	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12158	{
12159	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12160	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12161
12162	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12163	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12164	rc \|= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12165	AssertRCReturn(rc, rc);
12166
12167	VBOXSTRICTRC rcStrict;
12168	PVM pVM = pVCpu->CTX_SUFF(pVM);
12169	RTGCUINTPTR const uExitQual = pVmxTransient->uExitQual;
12170	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
12171	switch (uAccessType)
12172	{
12173	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
12174	{
12175	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
12176	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
12177	VMX_EXIT_QUAL_CRX_GENREG(uExitQual));
12178	AssertMsg( rcStrict == VINF_SUCCESS
12179	\|\| rcStrict == VINF_IEM_RAISED_XCPT
12180	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12181
12182	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
12183	{
12184	case 0:
12185	{
12186	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12187	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
12188	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
12189	Log4Func(("CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
12190
12191	/*
12192	* This is a kludge for handling switches back to real mode when we try to use
12193	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
12194	* deal with special selector values, so we have to return to ring-3 and run
12195	* there till the selector values are V86 mode compatible.
12196	*
12197	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
12198	* latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
12199	* at the end of this function.
12200	*/
12201	if ( rc == VINF_SUCCESS
12202	&& !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
12203	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
12204	&& (uOldCr0 & X86_CR0_PE)
12205	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
12206	{
12207	/** @todo check selectors rather than returning all the time. */
12208	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
12209	rcStrict = VINF_EM_RESCHEDULE_REM;
12210	}
12211	break;
12212	}
12213
12214	case 2:
12215	{
12216	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
12217	/* Nothing to do here, CR2 it's not part of the VMCS. */
12218	break;
12219	}
12220
12221	case 3:
12222	{
12223	Assert( !pVM->hm.s.fNestedPaging
12224	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
12225	\|\| pVCpu->hm.s.fUsingDebugLoop);
12226	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
12227	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12228	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
12229	Log4Func(("CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
12230	break;
12231	}
12232
12233	case 4:
12234	{
12235	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
12236	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12237	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
12238	Log4Func(("CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
12239	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12240	break;
12241	}
12242
12243	case 8:
12244	{
12245	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
12246	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
12247	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12248	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
12249	break;
12250	}
12251	default:
12252	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual)));
12253	break;
12254	}
12255	break;
12256	}
12257
12258	case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
12259	{
12260	Assert( !pVM->hm.s.fNestedPaging
12261	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
12262	\|\| pVCpu->hm.s.fUsingDebugLoop
12263	\|\| VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 3);
12264	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12265	Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 8
12266	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
12267
12268	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_GENREG(uExitQual),
12269	VMX_EXIT_QUAL_CRX_REGISTER(uExitQual));
12270	AssertMsg( rcStrict == VINF_SUCCESS
12271	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12272	#ifdef VBOX_WITH_STATISTICS
12273	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
12274	{
12275	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
12276	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
12277	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
12278	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
12279	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
12280	}
12281	#endif
12282	Log4Func(("CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
12283	VBOXSTRICTRC_VAL(rcStrict)));
12284	if (VMX_EXIT_QUAL_CRX_GENREG(uExitQual) == X86_GREG_xSP)
12285	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
12286	else
12287	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
12288	break;
12289	}
12290
12291	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12292	{
12293	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12294	AssertMsg( rcStrict == VINF_SUCCESS
12295	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12296
12297	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
12298	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12299	Log4Func(("CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12300	break;
12301	}
12302
12303	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12304	{
12305	/* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
12306	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
12307	AssertRCReturn(rc, rc);
12308	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual),
12309	pVmxTransient->uGuestLinearAddr);
12310	AssertMsg( rcStrict == VINF_SUCCESS
12311	\|\| rcStrict == VINF_IEM_RAISED_XCPT
12312	, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12313
12314	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
12315	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12316	Log4Func(("LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12317	break;
12318	}
12319
12320	default:
12321	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12322	VERR_VMX_UNEXPECTED_EXCEPTION);
12323	}
12324
12325	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
12326	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
12327	if (rcStrict == VINF_IEM_RAISED_XCPT)
12328	{
12329	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12330	rcStrict = VINF_SUCCESS;
12331	}
12332
12333	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12334	NOREF(pVM);
12335	return rcStrict;
12336	}
12337
12338
12339	/**
12340	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12341	* VM-exit.
12342	*/
12343	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12344	{
12345	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12346	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12347
12348	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12349	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12350	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12351	rc \|= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER);
12352	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12353	AssertRCReturn(rc, rc);
12354
12355	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12356	uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
12357	uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQual);
12358	bool fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12359	bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
12360	bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12361	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12362	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12363
12364	/*
12365	* Update exit history to see if this exit can be optimized.
12366	*/
12367	VBOXSTRICTRC rcStrict;
12368	PCEMEXITREC pExitRec = NULL;
12369	if ( !fGstStepping
12370	&& !fDbgStepping)
12371	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12372	!fIOString
12373	? !fIOWrite
12374	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12375	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12376	: !fIOWrite
12377	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12378	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12379	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12380	if (!pExitRec)
12381	{
12382	/* I/O operation lookup arrays. */
12383	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12384	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12385	uint32_t const cbValue = s_aIOSizes[uIOWidth];
12386	uint32_t const cbInstr = pVmxTransient->cbInstr;
12387	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12388	PVM pVM = pVCpu->CTX_SUFF(pVM);
12389	if (fIOString)
12390	{
12391	/*
12392	* INS/OUTS - I/O String instruction.
12393	*
12394	* Use instruction-information if available, otherwise fall back on
12395	* interpreting the instruction.
12396	*/
12397	Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12398	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
12399	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
12400	if (fInsOutsInfo)
12401	{
12402	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12403	AssertRCReturn(rc2, rc2);
12404	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12405	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12406	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12407	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
12408	if (fIOWrite)
12409	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12410	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
12411	else
12412	{
12413	/*
12414	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12415	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12416	* See Intel Instruction spec. for "INS".
12417	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12418	*/
12419	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
12420	}
12421	}
12422	else
12423	rcStrict = IEMExecOne(pVCpu);
12424
12425	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12426	fUpdateRipAlready = true;
12427	}
12428	else
12429	{
12430	/*
12431	* IN/OUT - I/O instruction.
12432	*/
12433	Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12434	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12435	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
12436	if (fIOWrite)
12437	{
12438	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
12439	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12440	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12441	&& !pCtx->eflags.Bits.u1TF)
12442	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
12443	}
12444	else
12445	{
12446	uint32_t u32Result = 0;
12447	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12448	if (IOM_SUCCESS(rcStrict))
12449	{
12450	/* Save result of I/O IN instr. in AL/AX/EAX. */
12451	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
12452	}
12453	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12454	&& !pCtx->eflags.Bits.u1TF)
12455	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
12456	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12457	}
12458	}
12459
12460	if (IOM_SUCCESS(rcStrict))
12461	{
12462	if (!fUpdateRipAlready)
12463	{
12464	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
12465	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12466	}
12467
12468	/*
12469	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12470	* while booting Fedora 17 64-bit guest.
12471	*
12472	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12473	*/
12474	if (fIOString)
12475	{
12476	/** @todo Single-step for INS/OUTS with REP prefix? */
12477	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12478	}
12479	else if ( !fDbgStepping
12480	&& fGstStepping)
12481	{
12482	rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12483	AssertRCReturn(rc, rc);
12484	}
12485
12486	/*
12487	* If any I/O breakpoints are armed, we need to check if one triggered
12488	* and take appropriate action.
12489	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12490	*/
12491	rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_DR7);
12492	AssertRCReturn(rc, rc);
12493
12494	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12495	* execution engines about whether hyper BPs and such are pending. */
12496	uint32_t const uDr7 = pCtx->dr[7];
12497	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12498	&& X86_DR7_ANY_RW_IO(uDr7)
12499	&& (pCtx->cr4 & X86_CR4_DE))
12500	\|\| DBGFBpIsHwIoArmed(pVM)))
12501	{
12502	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12503
12504	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12505	VMMRZCallRing3Disable(pVCpu);
12506	HM_DISABLE_PREEMPT(pVCpu);
12507
12508	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12509
12510	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
12511	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12512	{
12513	/* Raise #DB. */
12514	if (fIsGuestDbgActive)
12515	ASMSetDR6(pCtx->dr[6]);
12516	if (pCtx->dr[7] != uDr7)
12517	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
12518
12519	hmR0VmxSetPendingXcptDB(pVCpu);
12520	}
12521	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12522	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12523	else if ( rcStrict2 != VINF_SUCCESS
12524	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
12525	rcStrict = rcStrict2;
12526	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12527
12528	HM_RESTORE_PREEMPT();
12529	VMMRZCallRing3Enable(pVCpu);
12530	}
12531	}
12532
12533	#ifdef VBOX_STRICT
12534	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12535	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
12536	Assert(!fIOWrite);
12537	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12538	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
12539	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
12540	Assert(fIOWrite);
12541	else
12542	{
12543	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12544	* statuses, that the VMM device and some others may return. See
12545	* IOM_SUCCESS() for guidance. */
12546	AssertMsg( RT_FAILURE(rcStrict)
12547	\|\| rcStrict == VINF_SUCCESS
12548	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
12549	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
12550	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
12551	\|\| rcStrict == VINF_EM_RAW_TO_R3
12552	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12553	# endif
12554	}
12555	#endif
12556	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12557	}
12558	else
12559	{
12560	/*
12561	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
12562	*/
12563	int rc2 = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12564	AssertRCReturn(rc2, rc2);
12565	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12566	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12567	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12568	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12569	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
12570	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12571
12572	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12573	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12574
12575	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12576	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12577	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12578	}
12579	return rcStrict;
12580	}
12581
12582
12583	/**
12584	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12585	* VM-exit.
12586	*/
12587	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12588	{
12589	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12590
12591	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
12592	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12593	AssertRCReturn(rc, rc);
12594	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12595	{
12596	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12597	AssertRCReturn(rc, rc);
12598	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12599	{
12600	uint32_t uErrCode;
12601	RTGCUINTPTR GCPtrFaultAddress;
12602	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12603	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12604	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
12605	if (fErrorCodeValid)
12606	{
12607	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12608	AssertRCReturn(rc, rc);
12609	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12610	}
12611	else
12612	uErrCode = 0;
12613
12614	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12615	&& uVector == X86_XCPT_PF)
12616	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
12617	else
12618	GCPtrFaultAddress = 0;
12619
12620	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12621	AssertRCReturn(rc, rc);
12622
12623	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12624	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
12625
12626	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
12627	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12628	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12629	}
12630	}
12631
12632	/* Fall back to the interpreter to emulate the task-switch. */
12633	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12634	return VERR_EM_INTERPRETER;
12635	}
12636
12637
12638	/**
12639	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12640	*/
12641	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12642	{
12643	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12644	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG);
12645	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
12646	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12647	AssertRCReturn(rc, rc);
12648	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12649	return VINF_EM_DBG_STEPPED;
12650	}
12651
12652
12653	/**
12654	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12655	*/
12656	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12657	{
12658	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12659
12660	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12661
12662	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12663	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12664	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12665	{
12666	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12667	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12668	{
12669	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12670	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12671	}
12672	}
12673	else
12674	{
12675	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12676	rcStrict1 = VINF_SUCCESS;
12677	return rcStrict1;
12678	}
12679
12680	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12681	int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12682	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12683	AssertRCReturn(rc, rc);
12684
12685	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12686	uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
12687	VBOXSTRICTRC rcStrict2;
12688	switch (uAccessType)
12689	{
12690	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12691	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12692	{
12693	AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
12694	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
12695	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12696
12697	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12698	GCPhys &= PAGE_BASE_GC_MASK;
12699	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
12700	PVM pVM = pVCpu->CTX_SUFF(pVM);
12701	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12702	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
12703
12704	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12705	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12706	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12707	CPUMCTX2CORE(pCtx), GCPhys);
12708	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12709	if ( rcStrict2 == VINF_SUCCESS
12710	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12711	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
12712	{
12713	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
12714	\| HM_CHANGED_GUEST_APIC_TPR);
12715	rcStrict2 = VINF_SUCCESS;
12716	}
12717	break;
12718	}
12719
12720	default:
12721	Log4Func(("uAccessType=%#x\n", uAccessType));
12722	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12723	break;
12724	}
12725
12726	if (rcStrict2 != VINF_SUCCESS)
12727	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12728	return rcStrict2;
12729	}
12730
12731
12732	/**
12733	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12734	* VM-exit.
12735	*/
12736	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12737	{
12738	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12739
12740	/* We should -not- get this VM-exit if the guest's debug registers were active. */
12741	if (pVmxTransient->fWasGuestDebugStateActive)
12742	{
12743	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
12744	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12745	}
12746
12747	if ( !pVCpu->hm.s.fSingleInstruction
12748	&& !pVmxTransient->fWasHyperDebugStateActive)
12749	{
12750	Assert(!DBGFIsStepping(pVCpu));
12751	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12752
12753	/* Don't intercept MOV DRx any more. */
12754	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
12755	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12756	AssertRCReturn(rc, rc);
12757
12758	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12759	VMMRZCallRing3Disable(pVCpu);
12760	HM_DISABLE_PREEMPT(pVCpu);
12761
12762	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12763	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12764	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
12765
12766	HM_RESTORE_PREEMPT();
12767	VMMRZCallRing3Enable(pVCpu);
12768
12769	#ifdef VBOX_WITH_STATISTICS
12770	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12771	AssertRCReturn(rc, rc);
12772	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12773	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12774	else
12775	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12776	#endif
12777	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12778	return VINF_SUCCESS;
12779	}
12780
12781	/*
12782	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12783	* Update the segment registers and DR7 from the CPU.
12784	*/
12785	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12786	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12787	rc \|= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
12788	AssertRCReturn(rc, rc);
12789	Log4Func(("CS:RIP=%04x:%08RX64\n", pCtx->cs.Sel, pCtx->rip));
12790
12791	PVM pVM = pVCpu->CTX_SUFF(pVM);
12792	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12793	{
12794	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12795	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
12796	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
12797	if (RT_SUCCESS(rc))
12798	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12799	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12800	}
12801	else
12802	{
12803	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12804	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
12805	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
12806	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12807	}
12808
12809	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
12810	if (RT_SUCCESS(rc))
12811	{
12812	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12813	AssertRCReturn(rc2, rc2);
12814	return VINF_SUCCESS;
12815	}
12816	return rc;
12817	}
12818
12819
12820	/**
12821	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12822	* Conditional VM-exit.
12823	*/
12824	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12825	{
12826	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12827	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12828
12829	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12830	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12831	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12832	{
12833	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12834	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12835	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12836	{
12837	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12838	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12839	}
12840	}
12841	else
12842	{
12843	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12844	rcStrict1 = VINF_SUCCESS;
12845	return rcStrict1;
12846	}
12847
12848	/*
12849	* Get sufficent state and update the exit history entry.
12850	*/
12851	RTGCPHYS GCPhys;
12852	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12853	rc \|= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12854	AssertRCReturn(rc, rc);
12855
12856	VBOXSTRICTRC rcStrict;
12857	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12858	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
12859	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12860	if (!pExitRec)
12861	{
12862	/*
12863	* If we succeed, resume guest execution.
12864	* If we fail in interpreting the instruction because we couldn't get the guest physical address
12865	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12866	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12867	* weird case. See @bugref{6043}.
12868	*/
12869	PVM pVM = pVCpu->CTX_SUFF(pVM);
12870	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12871	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
12872	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12873	if ( rcStrict == VINF_SUCCESS
12874	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12875	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
12876	{
12877	/* Successfully handled MMIO operation. */
12878	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
12879	\| HM_CHANGED_GUEST_APIC_TPR);
12880	rcStrict = VINF_SUCCESS;
12881	}
12882	}
12883	else
12884	{
12885	/*
12886	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
12887	*/
12888	int rc2 = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12889	AssertRCReturn(rc2, rc2);
12890
12891	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12892	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12893
12894	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12895	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12896
12897	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12898	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12899	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12900	}
12901	return VBOXSTRICTRC_TODO(rcStrict);
12902	}
12903
12904
12905	/**
12906	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12907	* VM-exit.
12908	*/
12909	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12910	{
12911	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12912	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12913
12914	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12915	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12916	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12917	{
12918	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12919	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12920	Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12921	}
12922	else
12923	{
12924	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12925	rcStrict1 = VINF_SUCCESS;
12926	return rcStrict1;
12927	}
12928
12929	RTGCPHYS GCPhys;
12930	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12931	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12932	rc \|= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12933	AssertRCReturn(rc, rc);
12934
12935	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12936	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
12937
12938	RTGCUINT uErrorCode = 0;
12939	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12940	uErrorCode \|= X86_TRAP_PF_ID;
12941	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12942	uErrorCode \|= X86_TRAP_PF_RW;
12943	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12944	uErrorCode \|= X86_TRAP_PF_P;
12945
12946	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12947
12948
12949	/* Handle the pagefault trap for the nested shadow table. */
12950	PVM pVM = pVCpu->CTX_SUFF(pVM);
12951	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12952
12953	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
12954	pCtx->cs.Sel, pCtx->rip));
12955
12956	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
12957	TRPMResetTrap(pVCpu);
12958
12959	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12960	if ( rcStrict2 == VINF_SUCCESS
12961	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12962	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
12963	{
12964	/* Successfully synced our nested page tables. */
12965	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12966	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
12967	return VINF_SUCCESS;
12968	}
12969
12970	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12971	return rcStrict2;
12972	}
12973
12974	/** @} */
12975
12976	/** @name VM-exit exception handlers.
12977	* @{
12978	*/
12979	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12980	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12981	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12982
12983	/**
12984	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
12985	*/
12986	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12987	{
12988	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12989	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12990
12991	int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0);
12992	AssertRCReturn(rc, rc);
12993
12994	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
12995	{
12996	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12997	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12998
12999	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13000	* provides VM-exit instruction length. If this causes problem later,
13001	* disassemble the instruction like it's done on AMD-V. */
13002	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13003	AssertRCReturn(rc2, rc2);
13004	return rc;
13005	}
13006
13007	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13008	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13009	return rc;
13010	}
13011
13012
13013	/**
13014	* VM-exit exception handler for \#BP (Breakpoint exception).
13015	*/
13016	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13017	{
13018	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13019	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13020
13021	int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13022	AssertRCReturn(rc, rc);
13023
13024	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13025	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13026	if (rc == VINF_EM_RAW_GUEST_TRAP)
13027	{
13028	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13029	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13030	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13031	AssertRCReturn(rc, rc);
13032
13033	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13034	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13035	}
13036
13037	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
13038	return rc;
13039	}
13040
13041
13042	/**
13043	* VM-exit exception handler for \#AC (alignment check exception).
13044	*/
13045	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13046	{
13047	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13048
13049	/*
13050	* Re-inject it. We'll detect any nesting before getting here.
13051	*/
13052	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13053	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13054	AssertRCReturn(rc, rc);
13055	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13056
13057	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13058	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13059	return VINF_SUCCESS;
13060	}
13061
13062
13063	/**
13064	* VM-exit exception handler for \#DB (Debug exception).
13065	*/
13066	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13067	{
13068	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13069	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13070
13071	/*
13072	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13073	* for processing.
13074	*/
13075	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13076
13077	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13078	uint64_t uDR6 = X86_DR6_INIT_VAL;
13079	uDR6 \|= (pVmxTransient->uExitQual & (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
13080
13081	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13082	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13083	Log6Func(("rc=%Rrc\n", rc));
13084	if (rc == VINF_EM_RAW_GUEST_TRAP)
13085	{
13086	/*
13087	* The exception was for the guest. Update DR6, DR7.GD and
13088	* IA32_DEBUGCTL.LBR before forwarding it.
13089	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13090	*/
13091	VMMRZCallRing3Disable(pVCpu);
13092	HM_DISABLE_PREEMPT(pVCpu);
13093
13094	pCtx->dr[6] &= ~X86_DR6_B_MASK;
13095	pCtx->dr[6] \|= uDR6;
13096	if (CPUMIsGuestDebugStateActive(pVCpu))
13097	ASMSetDR6(pCtx->dr[6]);
13098
13099	HM_RESTORE_PREEMPT();
13100	VMMRZCallRing3Enable(pVCpu);
13101
13102	rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_DR7);
13103	AssertRCReturn(rc, rc);
13104
13105	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13106	pCtx->dr[7] &= ~X86_DR7_GD;
13107
13108	/* Paranoia. */
13109	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13110	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
13111
13112	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
13113	AssertRCReturn(rc, rc);
13114
13115	/*
13116	* Raise #DB in the guest.
13117	*
13118	* It is important to reflect exactly what the VM-exit gave us (preserving the
13119	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
13120	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
13121	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
13122	*
13123	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
13124	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
13125	*/
13126	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13127	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13128	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13129	AssertRCReturn(rc, rc);
13130	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13131	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13132	return VINF_SUCCESS;
13133	}
13134
13135	/*
13136	* Not a guest trap, must be a hypervisor related debug event then.
13137	* Update DR6 in case someone is interested in it.
13138	*/
13139	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13140	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13141	CPUMSetHyperDR6(pVCpu, uDR6);
13142
13143	return rc;
13144	}
13145
13146
13147	/**
13148	* VM-exit exception handler for \#GP (General-protection exception).
13149	*
13150	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13151	*/
13152	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13153	{
13154	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13155	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13156
13157	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13158	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13159	{ /* likely */ }
13160	else
13161	{
13162	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13163	Assert(pVCpu->hm.s.fUsingDebugLoop);
13164	#endif
13165	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13166	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13167	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13168	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13169	rc \|= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13170	AssertRCReturn(rc, rc);
13171	Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pCtx->cs.Sel, pCtx->rip,
13172	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
13173	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13174	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13175	return rc;
13176	}
13177
13178	Assert(CPUMIsGuestInRealModeEx(pCtx));
13179	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13180
13181	int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13182	AssertRCReturn(rc, rc);
13183
13184	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
13185	if (rcStrict == VINF_SUCCESS)
13186	{
13187	if (!CPUMIsGuestInRealModeEx(pCtx))
13188	{
13189	/*
13190	* The guest is no longer in real-mode, check if we can continue executing the
13191	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
13192	*/
13193	if (HMVmxCanExecuteGuest(pVCpu, pCtx))
13194	{
13195	Log4Func(("Mode changed but guest still suitable for executing using VT-x\n"));
13196	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
13197	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13198	}
13199	else
13200	{
13201	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
13202	rcStrict = VINF_EM_RESCHEDULE;
13203	}
13204	}
13205	else
13206	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13207	}
13208	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13209	{
13210	rcStrict = VINF_SUCCESS;
13211	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13212	}
13213	return VBOXSTRICTRC_VAL(rcStrict);
13214	}
13215
13216
13217	/**
13218	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13219	* the exception reported in the VMX transient structure back into the VM.
13220	*
13221	* @remarks Requires uExitIntInfo in the VMX transient structure to be
13222	* up-to-date.
13223	*/
13224	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13225	{
13226	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13227	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13228	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13229	("uVector=%#x u32XcptBitmap=%#X32\n",
13230	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13231	#endif
13232
13233	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13234	hmR0VmxCheckExitDueToEventDelivery(). */
13235	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13236	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13237	AssertRCReturn(rc, rc);
13238	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13239
13240	#ifdef DEBUG_ramshankar
13241	rc \|= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
13242	uint8_t uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13243	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13244	#endif
13245
13246	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13247	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13248	return VINF_SUCCESS;
13249	}
13250
13251
13252	/**
13253	* VM-exit exception handler for \#PF (Page-fault exception).
13254	*/
13255	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13256	{
13257	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13258	PVM pVM = pVCpu->CTX_SUFF(pVM);
13259	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13260	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13261	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13262	AssertRCReturn(rc, rc);
13263
13264	if (!pVM->hm.s.fNestedPaging)
13265	{ /* likely */ }
13266	else
13267	{
13268	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13269	Assert(pVCpu->hm.s.fUsingDebugLoop);
13270	#endif
13271	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13272	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13273	{
13274	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13275	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
13276	}
13277	else
13278	{
13279	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13280	hmR0VmxSetPendingXcptDF(pVCpu);
13281	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13282	}
13283	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13284	return rc;
13285	}
13286
13287	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13288	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13289	if (pVmxTransient->fVectoringPF)
13290	{
13291	Assert(pVCpu->hm.s.Event.fPending);
13292	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13293	}
13294
13295	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13296	rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13297	AssertRCReturn(rc, rc);
13298
13299	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
13300	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
13301
13302	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13303	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
13304
13305	Log4Func(("#PF: rc=%Rrc\n", rc));
13306	if (rc == VINF_SUCCESS)
13307	{
13308	/*
13309	* This is typically a shadow page table sync or a MMIO instruction. But we may have
13310	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13311	*/
13312	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13313	TRPMResetTrap(pVCpu);
13314	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13315	return rc;
13316	}
13317
13318	if (rc == VINF_EM_RAW_GUEST_TRAP)
13319	{
13320	if (!pVmxTransient->fVectoringDoublePF)
13321	{
13322	/* It's a guest page fault and needs to be reflected to the guest. */
13323	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13324	TRPMResetTrap(pVCpu);
13325	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13326	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13327	uGstErrorCode, pVmxTransient->uExitQual);
13328	}
13329	else
13330	{
13331	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13332	TRPMResetTrap(pVCpu);
13333	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13334	hmR0VmxSetPendingXcptDF(pVCpu);
13335	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13336	}
13337
13338	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13339	return VINF_SUCCESS;
13340	}
13341
13342	TRPMResetTrap(pVCpu);
13343	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13344	return rc;
13345	}
13346
13347	/** @} */
13348
13349	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13350	/** @name Nested-guest VM-exit handlers.
13351	* @{
13352	*/
13353	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13354	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= Nested-guest VM-exit handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13355	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13356
13357	/**
13358	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
13359	*/
13360	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13361	{
13362	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13363
13364	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13365	rc \|= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
13366	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13367	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13368	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13369	AssertRCReturn(rc, rc);
13370
13371	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13372
13373	VMXVEXITINFO ExitInfo;
13374	RT_ZERO(ExitInfo);
13375	ExitInfo.uReason = pVmxTransient->uExitReason;
13376	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13377	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13378	ExitInfo.cbInstr = pVmxTransient->cbInstr;
13379	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13380
13381	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
13382	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13383	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
13384	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13385	{
13386	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13387	rcStrict = VINF_SUCCESS;
13388	}
13389	return rcStrict;
13390	}
13391
13392
13393	/**
13394	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
13395	*/
13396	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13397	{
13398	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13399
13400	/** @todo NSTVMX: Vmlaunch. */
13401	hmR0VmxSetPendingXcptUD(pVCpu);
13402	return VINF_SUCCESS;
13403	}
13404
13405
13406	/**
13407	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
13408	*/
13409	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13410	{
13411	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13412
13413	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13414	rc \|= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
13415	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13416	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13417	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13418	AssertRCReturn(rc, rc);
13419
13420	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13421
13422	VMXVEXITINFO ExitInfo;
13423	RT_ZERO(ExitInfo);
13424	ExitInfo.uReason = pVmxTransient->uExitReason;
13425	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13426	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13427	ExitInfo.cbInstr = pVmxTransient->cbInstr;
13428	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13429
13430	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
13431	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13432	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
13433	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13434	{
13435	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13436	rcStrict = VINF_SUCCESS;
13437	}
13438	return rcStrict;
13439	}
13440
13441
13442	/**
13443	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
13444	*/
13445	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13446	{
13447	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13448
13449	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13450	rc \|= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
13451	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13452	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13453	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13454	AssertRCReturn(rc, rc);
13455
13456	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13457
13458	VMXVEXITINFO ExitInfo;
13459	RT_ZERO(ExitInfo);
13460	ExitInfo.uReason = pVmxTransient->uExitReason;
13461	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13462	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13463	ExitInfo.cbInstr = pVmxTransient->cbInstr;
13464	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
13465
13466	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
13467	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13468	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
13469	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13470	{
13471	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13472	rcStrict = VINF_SUCCESS;
13473	}
13474	return rcStrict;
13475	}
13476
13477
13478	/**
13479	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Unconditional VM-exit.
13480	*/
13481	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13482	{
13483	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13484
13485	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13486	rc \|= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
13487	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13488	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13489	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13490	AssertRCReturn(rc, rc);
13491
13492	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13493
13494	VMXVEXITINFO ExitInfo;
13495	RT_ZERO(ExitInfo);
13496	ExitInfo.uReason = pVmxTransient->uExitReason;
13497	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13498	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13499	ExitInfo.cbInstr = pVmxTransient->cbInstr;
13500	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
13501	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
13502
13503	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
13504	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13505	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
13506	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13507	{
13508	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13509	rcStrict = VINF_SUCCESS;
13510	}
13511	return rcStrict;
13512	}
13513
13514
13515	/**
13516	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
13517	*/
13518	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13519	{
13520	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13521
13522	/** @todo NSTVMX: Vmresume. */
13523	hmR0VmxSetPendingXcptUD(pVCpu);
13524	return VINF_SUCCESS;
13525	}
13526
13527
13528	/**
13529	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Unconditional VM-exit.
13530	*/
13531	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13532	{
13533	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13534
13535	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13536	rc \|= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
13537	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13538	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13539	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13540	AssertRCReturn(rc, rc);
13541
13542	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13543
13544	VMXVEXITINFO ExitInfo;
13545	RT_ZERO(ExitInfo);
13546	ExitInfo.uReason = pVmxTransient->uExitReason;
13547	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13548	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13549	ExitInfo.cbInstr = pVmxTransient->cbInstr;
13550	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
13551	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13552
13553	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
13554	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13555	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
13556	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13557	{
13558	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13559	rcStrict = VINF_SUCCESS;
13560	}
13561	return rcStrict;
13562	}
13563
13564
13565	/**
13566	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
13567	*/
13568	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13569	{
13570	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13571
13572	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13573	rc \|= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13574	AssertRCReturn(rc, rc);
13575
13576	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13577
13578	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
13579	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13580	{
13581	/* VMXOFF on success changes the internal hwvirt state but not anything that's visible to the guest. */
13582	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
13583	}
13584	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13585	{
13586	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13587	rcStrict = VINF_SUCCESS;
13588	}
13589	return rcStrict;
13590	}
13591
13592
13593	/**
13594	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
13595	*/
13596	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13597	{
13598	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13599
13600	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13601	rc \|= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
13602	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13603	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13604	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13605	AssertRCReturn(rc, rc);
13606
13607	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13608
13609	VMXVEXITINFO ExitInfo;
13610	RT_ZERO(ExitInfo);
13611	ExitInfo.uReason = pVmxTransient->uExitReason;
13612	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13613	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13614	ExitInfo.cbInstr = pVmxTransient->cbInstr;
13615	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13616
13617	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
13618	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13619	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
13620	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13621	{
13622	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13623	rcStrict = VINF_SUCCESS;
13624	}
13625	return rcStrict;
13626	}
13627
13628	/** @} */
13629	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13630

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 74603

Download in other formats: