VMXAllTemplate.cpp.h@ 92451

Last change on this file since 92451 was 92451, checked in by vboxsync, 3 years ago
VMM/VMXAllTemplate.cpp.h: Some more fixes for the NEM/darwin case, bugref:10136
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 606.0 KB

Line
1	/* $Id: VMXAllTemplate.cpp.h 92451 2021-11-16 10:40:19Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Code template for our own hypervisor and the NEM darwin backend using Apple's Hypervisor.framework.
4	*/
5
6	/*
7	* Copyright (C) 2012-2021 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Defined Constants And Macros *
21	*********************************************************************************************************************************/
22	#if !defined(VMX_VMCS_WRITE_16) \|\| !defined(VMX_VMCS_WRITE_32) \|\| !defined(VMX_VMCS_WRITE_64) \|\| !defined(VMX_VMCS_WRITE_64)
23	# error "At least one of the VMX_VMCS_WRITE_16, VMX_VMCS_WRITE_32, VMX_VMCS_WRITE_64 or VMX_VMCS_WRITE_64 is missing"
24	#endif
25
26
27	#if !defined(VMX_VMCS_READ_16) \|\| !defined(VMX_VMCS_READ_32) \|\| !defined(VMX_VMCS_READ_64) \|\| !defined(VMX_VMCS_READ_64)
28	# error "At least one of the VMX_VMCS_READ_16, VMX_VMCS_READ_32, VMX_VMCS_READ_64 or VMX_VMCS_READ_64 is missing"
29	#endif
30
31
32	/** Use the function table. */
33	#define HMVMX_USE_FUNCTION_TABLE
34
35	/** Determine which tagged-TLB flush handler to use. */
36	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
37	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
38	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
39	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
40
41	/**
42	* Flags to skip redundant reads of some common VMCS fields that are not part of
43	* the guest-CPU or VCPU state but are needed while handling VM-exits.
44	*/
45	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
46	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
47	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
48	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
49	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
50	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
51	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
52	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
53	#define HMVMX_READ_GUEST_PHYSICAL_ADDR RT_BIT_32(8)
54	#define HMVMX_READ_GUEST_PENDING_DBG_XCPTS RT_BIT_32(9)
55
56	/** All the VMCS fields required for processing of exception/NMI VM-exits. */
57	#define HMVMX_READ_XCPT_INFO ( HMVMX_READ_EXIT_INTERRUPTION_INFO \
58	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE \
59	\| HMVMX_READ_EXIT_INSTR_LEN \
60	\| HMVMX_READ_IDT_VECTORING_INFO \
61	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE)
62
63	/** Assert that all the given fields have been read from the VMCS. */
64	#ifdef VBOX_STRICT
65	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) \
66	do { \
67	uint32_t const fVmcsFieldRead = ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead); \
68	Assert((fVmcsFieldRead & (a_fReadFields)) == (a_fReadFields)); \
69	} while (0)
70	#else
71	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) do { } while (0)
72	#endif
73
74	/**
75	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
76	* guest using hardware-assisted VMX.
77	*
78	* This excludes state like GPRs (other than RSP) which are always are
79	* swapped and restored across the world-switch and also registers like EFER,
80	* MSR which cannot be modified by the guest without causing a VM-exit.
81	*/
82	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
83	\| CPUMCTX_EXTRN_RFLAGS \
84	\| CPUMCTX_EXTRN_RSP \
85	\| CPUMCTX_EXTRN_SREG_MASK \
86	\| CPUMCTX_EXTRN_TABLE_MASK \
87	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
88	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
89	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
90	\| CPUMCTX_EXTRN_TSC_AUX \
91	\| CPUMCTX_EXTRN_OTHER_MSRS \
92	\| CPUMCTX_EXTRN_CR0 \
93	\| CPUMCTX_EXTRN_CR3 \
94	\| CPUMCTX_EXTRN_CR4 \
95	\| CPUMCTX_EXTRN_DR7 \
96	\| CPUMCTX_EXTRN_HWVIRT \
97	\| CPUMCTX_EXTRN_HM_VMX_MASK)
98
99	/**
100	* Exception bitmap mask for real-mode guests (real-on-v86).
101	*
102	* We need to intercept all exceptions manually except:
103	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
104	* due to bugs in Intel CPUs.
105	* - \#PF need not be intercepted even in real-mode if we have nested paging
106	* support.
107	*/
108	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
109	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
110	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
111	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
112	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
113	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
114	\| RT_BIT(X86_XCPT_XF))
115
116	/** Maximum VM-instruction error number. */
117	#define HMVMX_INSTR_ERROR_MAX 28
118
119	/** Profiling macro. */
120	#ifdef HM_PROFILE_EXIT_DISPATCH
121	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitDispatch, ed)
122	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitDispatch, ed)
123	#else
124	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
125	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
126	#endif
127
128	#ifdef IN_RING0
129	/** Assert that preemption is disabled or covered by thread-context hooks. */
130	# define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
131	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
132
133	/** Assert that we haven't migrated CPUs when thread-context hooks are not
134	* used. */
135	# define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
136	\|\| (a_pVCpu)->hmr0.s.idEnteredCpu == RTMpCpuId(), \
137	("Illegal migration! Entered on CPU %u Current %u\n", \
138	(a_pVCpu)->hmr0.s.idEnteredCpu, RTMpCpuId()))
139	#else
140	# define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) do { } while (0)
141	# define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) do { } while (0)
142	#endif
143
144	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
145	* context. */
146	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
147	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
148	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
149
150	/** Log the VM-exit reason with an easily visible marker to identify it in a
151	* potential sea of logging data. */
152	#define HMVMX_LOG_EXIT(a_pVCpu, a_uExitReason) \
153	do { \
154	Log4(("VM-exit: vcpu[%RU32] %85s -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu, \
155	HMGetVmxExitName(a_uExitReason))); \
156	} while (0) \
157
158
159	/*********************************************************************************************************************************
160	* Structures and Typedefs *
161	*********************************************************************************************************************************/
162	/**
163	* Memory operand read or write access.
164	*/
165	typedef enum VMXMEMACCESS
166	{
167	VMXMEMACCESS_READ = 0,
168	VMXMEMACCESS_WRITE = 1
169	} VMXMEMACCESS;
170
171
172	/**
173	* VMX VM-exit handler.
174	*
175	* @returns Strict VBox status code (i.e. informational status codes too).
176	* @param pVCpu The cross context virtual CPU structure.
177	* @param pVmxTransient The VMX-transient structure.
178	*/
179	#ifndef HMVMX_USE_FUNCTION_TABLE
180	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
181	#else
182	typedef DECLCALLBACKTYPE(VBOXSTRICTRC, FNVMXEXITHANDLER,(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient));
183	/** Pointer to VM-exit handler. */
184	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
185	#endif
186
187	/**
188	* VMX VM-exit handler, non-strict status code.
189	*
190	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
191	*
192	* @returns VBox status code, no informational status code returned.
193	* @param pVCpu The cross context virtual CPU structure.
194	* @param pVmxTransient The VMX-transient structure.
195	*
196	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
197	* use of that status code will be replaced with VINF_EM_SOMETHING
198	* later when switching over to IEM.
199	*/
200	#ifndef HMVMX_USE_FUNCTION_TABLE
201	typedef int FNVMXEXITHANDLERNSRC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
202	#else
203	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
204	#endif
205
206
207	/*********************************************************************************************************************************
208	* Internal Functions *
209	*********************************************************************************************************************************/
210	#ifndef HMVMX_USE_FUNCTION_TABLE
211	DECLINLINE(VBOXSTRICTRC) vmxHCHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
212	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
213	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
214	#else
215	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
216	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
217	#endif
218	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
219	DECLINLINE(VBOXSTRICTRC) vmxHCHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
220	#endif
221
222	static int vmxHCImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
223
224	/** @name VM-exit handler prototypes.
225	* @{
226	*/
227	static FNVMXEXITHANDLER vmxHCExitXcptOrNmi;
228	static FNVMXEXITHANDLER vmxHCExitExtInt;
229	static FNVMXEXITHANDLER vmxHCExitTripleFault;
230	static FNVMXEXITHANDLERNSRC vmxHCExitIntWindow;
231	static FNVMXEXITHANDLERNSRC vmxHCExitNmiWindow;
232	static FNVMXEXITHANDLER vmxHCExitTaskSwitch;
233	static FNVMXEXITHANDLER vmxHCExitCpuid;
234	static FNVMXEXITHANDLER vmxHCExitGetsec;
235	static FNVMXEXITHANDLER vmxHCExitHlt;
236	static FNVMXEXITHANDLERNSRC vmxHCExitInvd;
237	static FNVMXEXITHANDLER vmxHCExitInvlpg;
238	static FNVMXEXITHANDLER vmxHCExitRdpmc;
239	static FNVMXEXITHANDLER vmxHCExitVmcall;
240	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
241	static FNVMXEXITHANDLER vmxHCExitVmclear;
242	static FNVMXEXITHANDLER vmxHCExitVmlaunch;
243	static FNVMXEXITHANDLER vmxHCExitVmptrld;
244	static FNVMXEXITHANDLER vmxHCExitVmptrst;
245	static FNVMXEXITHANDLER vmxHCExitVmread;
246	static FNVMXEXITHANDLER vmxHCExitVmresume;
247	static FNVMXEXITHANDLER vmxHCExitVmwrite;
248	static FNVMXEXITHANDLER vmxHCExitVmxoff;
249	static FNVMXEXITHANDLER vmxHCExitVmxon;
250	static FNVMXEXITHANDLER vmxHCExitInvvpid;
251	#endif
252	static FNVMXEXITHANDLER vmxHCExitRdtsc;
253	static FNVMXEXITHANDLER vmxHCExitMovCRx;
254	static FNVMXEXITHANDLER vmxHCExitMovDRx;
255	static FNVMXEXITHANDLER vmxHCExitIoInstr;
256	static FNVMXEXITHANDLER vmxHCExitRdmsr;
257	static FNVMXEXITHANDLER vmxHCExitWrmsr;
258	static FNVMXEXITHANDLER vmxHCExitMwait;
259	static FNVMXEXITHANDLER vmxHCExitMtf;
260	static FNVMXEXITHANDLER vmxHCExitMonitor;
261	static FNVMXEXITHANDLER vmxHCExitPause;
262	static FNVMXEXITHANDLERNSRC vmxHCExitTprBelowThreshold;
263	static FNVMXEXITHANDLER vmxHCExitApicAccess;
264	static FNVMXEXITHANDLER vmxHCExitEptViolation;
265	static FNVMXEXITHANDLER vmxHCExitEptMisconfig;
266	static FNVMXEXITHANDLER vmxHCExitRdtscp;
267	static FNVMXEXITHANDLER vmxHCExitPreemptTimer;
268	static FNVMXEXITHANDLERNSRC vmxHCExitWbinvd;
269	static FNVMXEXITHANDLER vmxHCExitXsetbv;
270	static FNVMXEXITHANDLER vmxHCExitInvpcid;
271	static FNVMXEXITHANDLERNSRC vmxHCExitSetPendingXcptUD;
272	static FNVMXEXITHANDLERNSRC vmxHCExitErrInvalidGuestState;
273	static FNVMXEXITHANDLERNSRC vmxHCExitErrUnexpected;
274	/** @} */
275
276	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
277	/** @name Nested-guest VM-exit handler prototypes.
278	* @{
279	*/
280	static FNVMXEXITHANDLER vmxHCExitXcptOrNmiNested;
281	static FNVMXEXITHANDLER vmxHCExitTripleFaultNested;
282	static FNVMXEXITHANDLERNSRC vmxHCExitIntWindowNested;
283	static FNVMXEXITHANDLERNSRC vmxHCExitNmiWindowNested;
284	static FNVMXEXITHANDLER vmxHCExitTaskSwitchNested;
285	static FNVMXEXITHANDLER vmxHCExitHltNested;
286	static FNVMXEXITHANDLER vmxHCExitInvlpgNested;
287	static FNVMXEXITHANDLER vmxHCExitRdpmcNested;
288	static FNVMXEXITHANDLER vmxHCExitVmreadVmwriteNested;
289	static FNVMXEXITHANDLER vmxHCExitRdtscNested;
290	static FNVMXEXITHANDLER vmxHCExitMovCRxNested;
291	static FNVMXEXITHANDLER vmxHCExitMovDRxNested;
292	static FNVMXEXITHANDLER vmxHCExitIoInstrNested;
293	static FNVMXEXITHANDLER vmxHCExitRdmsrNested;
294	static FNVMXEXITHANDLER vmxHCExitWrmsrNested;
295	static FNVMXEXITHANDLER vmxHCExitMwaitNested;
296	static FNVMXEXITHANDLER vmxHCExitMtfNested;
297	static FNVMXEXITHANDLER vmxHCExitMonitorNested;
298	static FNVMXEXITHANDLER vmxHCExitPauseNested;
299	static FNVMXEXITHANDLERNSRC vmxHCExitTprBelowThresholdNested;
300	static FNVMXEXITHANDLER vmxHCExitApicAccessNested;
301	static FNVMXEXITHANDLER vmxHCExitApicWriteNested;
302	static FNVMXEXITHANDLER vmxHCExitVirtEoiNested;
303	static FNVMXEXITHANDLER vmxHCExitRdtscpNested;
304	static FNVMXEXITHANDLERNSRC vmxHCExitWbinvdNested;
305	static FNVMXEXITHANDLER vmxHCExitInvpcidNested;
306	static FNVMXEXITHANDLERNSRC vmxHCExitErrInvalidGuestStateNested;
307	static FNVMXEXITHANDLER vmxHCExitInstrNested;
308	static FNVMXEXITHANDLER vmxHCExitInstrWithInfoNested;
309	/** @} */
310	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
311
312
313	/*********************************************************************************************************************************
314	* Global Variables *
315	*********************************************************************************************************************************/
316	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
317	/**
318	* Array of all VMCS fields.
319	* Any fields added to the VT-x spec. should be added here.
320	*
321	* Currently only used to derive shadow VMCS fields for hardware-assisted execution
322	* of nested-guests.
323	*/
324	static const uint32_t g_aVmcsFields[] =
325	{
326	/* 16-bit control fields. */
327	VMX_VMCS16_VPID,
328	VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR,
329	VMX_VMCS16_EPTP_INDEX,
330
331	/* 16-bit guest-state fields. */
332	VMX_VMCS16_GUEST_ES_SEL,
333	VMX_VMCS16_GUEST_CS_SEL,
334	VMX_VMCS16_GUEST_SS_SEL,
335	VMX_VMCS16_GUEST_DS_SEL,
336	VMX_VMCS16_GUEST_FS_SEL,
337	VMX_VMCS16_GUEST_GS_SEL,
338	VMX_VMCS16_GUEST_LDTR_SEL,
339	VMX_VMCS16_GUEST_TR_SEL,
340	VMX_VMCS16_GUEST_INTR_STATUS,
341	VMX_VMCS16_GUEST_PML_INDEX,
342
343	/* 16-bits host-state fields. */
344	VMX_VMCS16_HOST_ES_SEL,
345	VMX_VMCS16_HOST_CS_SEL,
346	VMX_VMCS16_HOST_SS_SEL,
347	VMX_VMCS16_HOST_DS_SEL,
348	VMX_VMCS16_HOST_FS_SEL,
349	VMX_VMCS16_HOST_GS_SEL,
350	VMX_VMCS16_HOST_TR_SEL,
351
352	/* 64-bit control fields. */
353	VMX_VMCS64_CTRL_IO_BITMAP_A_FULL,
354	VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH,
355	VMX_VMCS64_CTRL_IO_BITMAP_B_FULL,
356	VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH,
357	VMX_VMCS64_CTRL_MSR_BITMAP_FULL,
358	VMX_VMCS64_CTRL_MSR_BITMAP_HIGH,
359	VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL,
360	VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH,
361	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,
362	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH,
363	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL,
364	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH,
365	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL,
366	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH,
367	VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL,
368	VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH,
369	VMX_VMCS64_CTRL_TSC_OFFSET_FULL,
370	VMX_VMCS64_CTRL_TSC_OFFSET_HIGH,
371	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL,
372	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH,
373	VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL,
374	VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH,
375	VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL,
376	VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH,
377	VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL,
378	VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH,
379	VMX_VMCS64_CTRL_EPTP_FULL,
380	VMX_VMCS64_CTRL_EPTP_HIGH,
381	VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL,
382	VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH,
383	VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL,
384	VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH,
385	VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL,
386	VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH,
387	VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL,
388	VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH,
389	VMX_VMCS64_CTRL_EPTP_LIST_FULL,
390	VMX_VMCS64_CTRL_EPTP_LIST_HIGH,
391	VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL,
392	VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH,
393	VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL,
394	VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH,
395	VMX_VMCS64_CTRL_VE_XCPT_INFO_ADDR_FULL,
396	VMX_VMCS64_CTRL_VE_XCPT_INFO_ADDR_HIGH,
397	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL,
398	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH,
399	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL,
400	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH,
401	VMX_VMCS64_CTRL_SPPTP_FULL,
402	VMX_VMCS64_CTRL_SPPTP_HIGH,
403	VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL,
404	VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH,
405	VMX_VMCS64_CTRL_PROC_EXEC3_FULL,
406	VMX_VMCS64_CTRL_PROC_EXEC3_HIGH,
407	VMX_VMCS64_CTRL_ENCLV_EXITING_BITMAP_FULL,
408	VMX_VMCS64_CTRL_ENCLV_EXITING_BITMAP_HIGH,
409
410	/* 64-bit read-only data fields. */
411	VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,
412	VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH,
413
414	/* 64-bit guest-state fields. */
415	VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL,
416	VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH,
417	VMX_VMCS64_GUEST_DEBUGCTL_FULL,
418	VMX_VMCS64_GUEST_DEBUGCTL_HIGH,
419	VMX_VMCS64_GUEST_PAT_FULL,
420	VMX_VMCS64_GUEST_PAT_HIGH,
421	VMX_VMCS64_GUEST_EFER_FULL,
422	VMX_VMCS64_GUEST_EFER_HIGH,
423	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL,
424	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH,
425	VMX_VMCS64_GUEST_PDPTE0_FULL,
426	VMX_VMCS64_GUEST_PDPTE0_HIGH,
427	VMX_VMCS64_GUEST_PDPTE1_FULL,
428	VMX_VMCS64_GUEST_PDPTE1_HIGH,
429	VMX_VMCS64_GUEST_PDPTE2_FULL,
430	VMX_VMCS64_GUEST_PDPTE2_HIGH,
431	VMX_VMCS64_GUEST_PDPTE3_FULL,
432	VMX_VMCS64_GUEST_PDPTE3_HIGH,
433	VMX_VMCS64_GUEST_BNDCFGS_FULL,
434	VMX_VMCS64_GUEST_BNDCFGS_HIGH,
435	VMX_VMCS64_GUEST_RTIT_CTL_FULL,
436	VMX_VMCS64_GUEST_RTIT_CTL_HIGH,
437	VMX_VMCS64_GUEST_PKRS_FULL,
438	VMX_VMCS64_GUEST_PKRS_HIGH,
439
440	/* 64-bit host-state fields. */
441	VMX_VMCS64_HOST_PAT_FULL,
442	VMX_VMCS64_HOST_PAT_HIGH,
443	VMX_VMCS64_HOST_EFER_FULL,
444	VMX_VMCS64_HOST_EFER_HIGH,
445	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL,
446	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH,
447	VMX_VMCS64_HOST_PKRS_FULL,
448	VMX_VMCS64_HOST_PKRS_HIGH,
449
450	/* 32-bit control fields. */
451	VMX_VMCS32_CTRL_PIN_EXEC,
452	VMX_VMCS32_CTRL_PROC_EXEC,
453	VMX_VMCS32_CTRL_EXCEPTION_BITMAP,
454	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,
455	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,
456	VMX_VMCS32_CTRL_CR3_TARGET_COUNT,
457	VMX_VMCS32_CTRL_EXIT,
458	VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,
459	VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,
460	VMX_VMCS32_CTRL_ENTRY,
461	VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,
462	VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,
463	VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,
464	VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,
465	VMX_VMCS32_CTRL_TPR_THRESHOLD,
466	VMX_VMCS32_CTRL_PROC_EXEC2,
467	VMX_VMCS32_CTRL_PLE_GAP,
468	VMX_VMCS32_CTRL_PLE_WINDOW,
469
470	/* 32-bits read-only fields. */
471	VMX_VMCS32_RO_VM_INSTR_ERROR,
472	VMX_VMCS32_RO_EXIT_REASON,
473	VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,
474	VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE,
475	VMX_VMCS32_RO_IDT_VECTORING_INFO,
476	VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,
477	VMX_VMCS32_RO_EXIT_INSTR_LENGTH,
478	VMX_VMCS32_RO_EXIT_INSTR_INFO,
479
480	/* 32-bit guest-state fields. */
481	VMX_VMCS32_GUEST_ES_LIMIT,
482	VMX_VMCS32_GUEST_CS_LIMIT,
483	VMX_VMCS32_GUEST_SS_LIMIT,
484	VMX_VMCS32_GUEST_DS_LIMIT,
485	VMX_VMCS32_GUEST_FS_LIMIT,
486	VMX_VMCS32_GUEST_GS_LIMIT,
487	VMX_VMCS32_GUEST_LDTR_LIMIT,
488	VMX_VMCS32_GUEST_TR_LIMIT,
489	VMX_VMCS32_GUEST_GDTR_LIMIT,
490	VMX_VMCS32_GUEST_IDTR_LIMIT,
491	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
492	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
493	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
494	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
495	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
496	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS,
497	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS,
498	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS,
499	VMX_VMCS32_GUEST_INT_STATE,
500	VMX_VMCS32_GUEST_ACTIVITY_STATE,
501	VMX_VMCS32_GUEST_SMBASE,
502	VMX_VMCS32_GUEST_SYSENTER_CS,
503	VMX_VMCS32_PREEMPT_TIMER_VALUE,
504
505	/* 32-bit host-state fields. */
506	VMX_VMCS32_HOST_SYSENTER_CS,
507
508	/* Natural-width control fields. */
509	VMX_VMCS_CTRL_CR0_MASK,
510	VMX_VMCS_CTRL_CR4_MASK,
511	VMX_VMCS_CTRL_CR0_READ_SHADOW,
512	VMX_VMCS_CTRL_CR4_READ_SHADOW,
513	VMX_VMCS_CTRL_CR3_TARGET_VAL0,
514	VMX_VMCS_CTRL_CR3_TARGET_VAL1,
515	VMX_VMCS_CTRL_CR3_TARGET_VAL2,
516	VMX_VMCS_CTRL_CR3_TARGET_VAL3,
517
518	/* Natural-width read-only data fields. */
519	VMX_VMCS_RO_EXIT_QUALIFICATION,
520	VMX_VMCS_RO_IO_RCX,
521	VMX_VMCS_RO_IO_RSI,
522	VMX_VMCS_RO_IO_RDI,
523	VMX_VMCS_RO_IO_RIP,
524	VMX_VMCS_RO_GUEST_LINEAR_ADDR,
525
526	/* Natural-width guest-state field */
527	VMX_VMCS_GUEST_CR0,
528	VMX_VMCS_GUEST_CR3,
529	VMX_VMCS_GUEST_CR4,
530	VMX_VMCS_GUEST_ES_BASE,
531	VMX_VMCS_GUEST_CS_BASE,
532	VMX_VMCS_GUEST_SS_BASE,
533	VMX_VMCS_GUEST_DS_BASE,
534	VMX_VMCS_GUEST_FS_BASE,
535	VMX_VMCS_GUEST_GS_BASE,
536	VMX_VMCS_GUEST_LDTR_BASE,
537	VMX_VMCS_GUEST_TR_BASE,
538	VMX_VMCS_GUEST_GDTR_BASE,
539	VMX_VMCS_GUEST_IDTR_BASE,
540	VMX_VMCS_GUEST_DR7,
541	VMX_VMCS_GUEST_RSP,
542	VMX_VMCS_GUEST_RIP,
543	VMX_VMCS_GUEST_RFLAGS,
544	VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
545	VMX_VMCS_GUEST_SYSENTER_ESP,
546	VMX_VMCS_GUEST_SYSENTER_EIP,
547	VMX_VMCS_GUEST_S_CET,
548	VMX_VMCS_GUEST_SSP,
549	VMX_VMCS_GUEST_INTR_SSP_TABLE_ADDR,
550
551	/* Natural-width host-state fields */
552	VMX_VMCS_HOST_CR0,
553	VMX_VMCS_HOST_CR3,
554	VMX_VMCS_HOST_CR4,
555	VMX_VMCS_HOST_FS_BASE,
556	VMX_VMCS_HOST_GS_BASE,
557	VMX_VMCS_HOST_TR_BASE,
558	VMX_VMCS_HOST_GDTR_BASE,
559	VMX_VMCS_HOST_IDTR_BASE,
560	VMX_VMCS_HOST_SYSENTER_ESP,
561	VMX_VMCS_HOST_SYSENTER_EIP,
562	VMX_VMCS_HOST_RSP,
563	VMX_VMCS_HOST_RIP,
564	VMX_VMCS_HOST_S_CET,
565	VMX_VMCS_HOST_SSP,
566	VMX_VMCS_HOST_INTR_SSP_TABLE_ADDR
567	};
568	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
569
570	#ifdef VBOX_STRICT
571	static const uint32_t g_aVmcsSegBase[] =
572	{
573	VMX_VMCS_GUEST_ES_BASE,
574	VMX_VMCS_GUEST_CS_BASE,
575	VMX_VMCS_GUEST_SS_BASE,
576	VMX_VMCS_GUEST_DS_BASE,
577	VMX_VMCS_GUEST_FS_BASE,
578	VMX_VMCS_GUEST_GS_BASE
579	};
580	static const uint32_t g_aVmcsSegSel[] =
581	{
582	VMX_VMCS16_GUEST_ES_SEL,
583	VMX_VMCS16_GUEST_CS_SEL,
584	VMX_VMCS16_GUEST_SS_SEL,
585	VMX_VMCS16_GUEST_DS_SEL,
586	VMX_VMCS16_GUEST_FS_SEL,
587	VMX_VMCS16_GUEST_GS_SEL
588	};
589	static const uint32_t g_aVmcsSegLimit[] =
590	{
591	VMX_VMCS32_GUEST_ES_LIMIT,
592	VMX_VMCS32_GUEST_CS_LIMIT,
593	VMX_VMCS32_GUEST_SS_LIMIT,
594	VMX_VMCS32_GUEST_DS_LIMIT,
595	VMX_VMCS32_GUEST_FS_LIMIT,
596	VMX_VMCS32_GUEST_GS_LIMIT
597	};
598	static const uint32_t g_aVmcsSegAttr[] =
599	{
600	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
601	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
602	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
603	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
604	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
605	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
606	};
607	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
608	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
609	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
610	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
611	#endif /* VBOX_STRICT */
612
613	#ifdef HMVMX_USE_FUNCTION_TABLE
614	/**
615	* VMX_EXIT dispatch table.
616	*/
617	static const struct CLANG11NOTHROWWEIRDNESS { PFNVMXEXITHANDLER pfn; } g_aVMExitHandlers[VMX_EXIT_MAX + 1] =
618	{
619	/* 0 VMX_EXIT_XCPT_OR_NMI */ { vmxHCExitXcptOrNmi },
620	/* 1 VMX_EXIT_EXT_INT */ { vmxHCExitExtInt },
621	/* 2 VMX_EXIT_TRIPLE_FAULT */ { vmxHCExitTripleFault },
622	/* 3 VMX_EXIT_INIT_SIGNAL */ { vmxHCExitErrUnexpected },
623	/* 4 VMX_EXIT_SIPI */ { vmxHCExitErrUnexpected },
624	/* 5 VMX_EXIT_IO_SMI */ { vmxHCExitErrUnexpected },
625	/* 6 VMX_EXIT_SMI */ { vmxHCExitErrUnexpected },
626	/* 7 VMX_EXIT_INT_WINDOW */ { vmxHCExitIntWindow },
627	/* 8 VMX_EXIT_NMI_WINDOW */ { vmxHCExitNmiWindow },
628	/* 9 VMX_EXIT_TASK_SWITCH */ { vmxHCExitTaskSwitch },
629	/* 10 VMX_EXIT_CPUID */ { vmxHCExitCpuid },
630	/* 11 VMX_EXIT_GETSEC */ { vmxHCExitGetsec },
631	/* 12 VMX_EXIT_HLT */ { vmxHCExitHlt },
632	/* 13 VMX_EXIT_INVD */ { vmxHCExitInvd },
633	/* 14 VMX_EXIT_INVLPG */ { vmxHCExitInvlpg },
634	/* 15 VMX_EXIT_RDPMC */ { vmxHCExitRdpmc },
635	/* 16 VMX_EXIT_RDTSC */ { vmxHCExitRdtsc },
636	/* 17 VMX_EXIT_RSM */ { vmxHCExitErrUnexpected },
637	/* 18 VMX_EXIT_VMCALL */ { vmxHCExitVmcall },
638	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
639	/* 19 VMX_EXIT_VMCLEAR */ { vmxHCExitVmclear },
640	/* 20 VMX_EXIT_VMLAUNCH */ { vmxHCExitVmlaunch },
641	/* 21 VMX_EXIT_VMPTRLD */ { vmxHCExitVmptrld },
642	/* 22 VMX_EXIT_VMPTRST */ { vmxHCExitVmptrst },
643	/* 23 VMX_EXIT_VMREAD */ { vmxHCExitVmread },
644	/* 24 VMX_EXIT_VMRESUME */ { vmxHCExitVmresume },
645	/* 25 VMX_EXIT_VMWRITE */ { vmxHCExitVmwrite },
646	/* 26 VMX_EXIT_VMXOFF */ { vmxHCExitVmxoff },
647	/* 27 VMX_EXIT_VMXON */ { vmxHCExitVmxon },
648	#else
649	/* 19 VMX_EXIT_VMCLEAR */ { vmxHCExitSetPendingXcptUD },
650	/* 20 VMX_EXIT_VMLAUNCH */ { vmxHCExitSetPendingXcptUD },
651	/* 21 VMX_EXIT_VMPTRLD */ { vmxHCExitSetPendingXcptUD },
652	/* 22 VMX_EXIT_VMPTRST */ { vmxHCExitSetPendingXcptUD },
653	/* 23 VMX_EXIT_VMREAD */ { vmxHCExitSetPendingXcptUD },
654	/* 24 VMX_EXIT_VMRESUME */ { vmxHCExitSetPendingXcptUD },
655	/* 25 VMX_EXIT_VMWRITE */ { vmxHCExitSetPendingXcptUD },
656	/* 26 VMX_EXIT_VMXOFF */ { vmxHCExitSetPendingXcptUD },
657	/* 27 VMX_EXIT_VMXON */ { vmxHCExitSetPendingXcptUD },
658	#endif
659	/* 28 VMX_EXIT_MOV_CRX */ { vmxHCExitMovCRx },
660	/* 29 VMX_EXIT_MOV_DRX */ { vmxHCExitMovDRx },
661	/* 30 VMX_EXIT_IO_INSTR */ { vmxHCExitIoInstr },
662	/* 31 VMX_EXIT_RDMSR */ { vmxHCExitRdmsr },
663	/* 32 VMX_EXIT_WRMSR */ { vmxHCExitWrmsr },
664	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ { vmxHCExitErrInvalidGuestState },
665	/* 34 VMX_EXIT_ERR_MSR_LOAD */ { vmxHCExitErrUnexpected },
666	/* 35 UNDEFINED */ { vmxHCExitErrUnexpected },
667	/* 36 VMX_EXIT_MWAIT */ { vmxHCExitMwait },
668	/* 37 VMX_EXIT_MTF */ { vmxHCExitMtf },
669	/* 38 UNDEFINED */ { vmxHCExitErrUnexpected },
670	/* 39 VMX_EXIT_MONITOR */ { vmxHCExitMonitor },
671	/* 40 VMX_EXIT_PAUSE */ { vmxHCExitPause },
672	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ { vmxHCExitErrUnexpected },
673	/* 42 UNDEFINED */ { vmxHCExitErrUnexpected },
674	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ { vmxHCExitTprBelowThreshold },
675	/* 44 VMX_EXIT_APIC_ACCESS */ { vmxHCExitApicAccess },
676	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ { vmxHCExitErrUnexpected },
677	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ { vmxHCExitErrUnexpected },
678	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ { vmxHCExitErrUnexpected },
679	/* 48 VMX_EXIT_EPT_VIOLATION */ { vmxHCExitEptViolation },
680	/* 49 VMX_EXIT_EPT_MISCONFIG */ { vmxHCExitEptMisconfig },
681	/* 50 VMX_EXIT_INVEPT */ { vmxHCExitSetPendingXcptUD },
682	/* 51 VMX_EXIT_RDTSCP */ { vmxHCExitRdtscp },
683	/* 52 VMX_EXIT_PREEMPT_TIMER */ { vmxHCExitPreemptTimer },
684	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
685	/* 53 VMX_EXIT_INVVPID */ { vmxHCExitInvvpid },
686	#else
687	/* 53 VMX_EXIT_INVVPID */ { vmxHCExitSetPendingXcptUD },
688	#endif
689	/* 54 VMX_EXIT_WBINVD */ { vmxHCExitWbinvd },
690	/* 55 VMX_EXIT_XSETBV */ { vmxHCExitXsetbv },
691	/* 56 VMX_EXIT_APIC_WRITE */ { vmxHCExitErrUnexpected },
692	/* 57 VMX_EXIT_RDRAND */ { vmxHCExitErrUnexpected },
693	/* 58 VMX_EXIT_INVPCID */ { vmxHCExitInvpcid },
694	/* 59 VMX_EXIT_VMFUNC */ { vmxHCExitErrUnexpected },
695	/* 60 VMX_EXIT_ENCLS */ { vmxHCExitErrUnexpected },
696	/* 61 VMX_EXIT_RDSEED */ { vmxHCExitErrUnexpected },
697	/* 62 VMX_EXIT_PML_FULL */ { vmxHCExitErrUnexpected },
698	/* 63 VMX_EXIT_XSAVES */ { vmxHCExitErrUnexpected },
699	/* 64 VMX_EXIT_XRSTORS */ { vmxHCExitErrUnexpected },
700	/* 65 UNDEFINED */ { vmxHCExitErrUnexpected },
701	/* 66 VMX_EXIT_SPP_EVENT */ { vmxHCExitErrUnexpected },
702	/* 67 VMX_EXIT_UMWAIT */ { vmxHCExitErrUnexpected },
703	/* 68 VMX_EXIT_TPAUSE */ { vmxHCExitErrUnexpected },
704	/* 69 VMX_EXIT_LOADIWKEY */ { vmxHCExitErrUnexpected },
705	};
706	#endif /* HMVMX_USE_FUNCTION_TABLE */
707
708	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
709	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
710	{
711	/* 0 */ "(Not Used)",
712	/* 1 */ "VMCALL executed in VMX root operation.",
713	/* 2 */ "VMCLEAR with invalid physical address.",
714	/* 3 */ "VMCLEAR with VMXON pointer.",
715	/* 4 */ "VMLAUNCH with non-clear VMCS.",
716	/* 5 */ "VMRESUME with non-launched VMCS.",
717	/* 6 */ "VMRESUME after VMXOFF",
718	/* 7 */ "VM-entry with invalid control fields.",
719	/* 8 */ "VM-entry with invalid host state fields.",
720	/* 9 */ "VMPTRLD with invalid physical address.",
721	/* 10 */ "VMPTRLD with VMXON pointer.",
722	/* 11 */ "VMPTRLD with incorrect revision identifier.",
723	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
724	/* 13 */ "VMWRITE to read-only VMCS component.",
725	/* 14 */ "(Not Used)",
726	/* 15 */ "VMXON executed in VMX root operation.",
727	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
728	/* 17 */ "VM-entry with non-launched executing VMCS.",
729	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
730	/* 19 */ "VMCALL with non-clear VMCS.",
731	/* 20 */ "VMCALL with invalid VM-exit control fields.",
732	/* 21 */ "(Not Used)",
733	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
734	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
735	/* 24 */ "VMCALL with invalid SMM-monitor features.",
736	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
737	/* 26 */ "VM-entry with events blocked by MOV SS.",
738	/* 27 */ "(Not Used)",
739	/* 28 */ "Invalid operand to INVEPT/INVVPID."
740	};
741	#endif /* VBOX_STRICT && LOG_ENABLED */
742
743
744	#ifdef IN_RING0
745	/**
746	* Checks if the given MSR is part of the lastbranch-from-IP MSR stack.
747	* @returns @c true if it's part of LBR stack, @c false otherwise.
748	*
749	* @param pVM The cross context VM structure.
750	* @param idMsr The MSR.
751	* @param pidxMsr Where to store the index of the MSR in the LBR MSR array.
752	* Optional, can be NULL.
753	*
754	* @remarks Must only be called when LBR is enabled.
755	*/
756	DECL_FORCE_INLINE(bool) vmxHCIsLbrBranchFromMsr(PCVMCC pVM, uint32_t idMsr, uint32_t *pidxMsr)
757	{
758	Assert(VM_IS_VMX_LBR(pVM));
759	Assert(pVM->hmr0.s.vmx.idLbrFromIpMsrFirst);
760	uint32_t const cLbrStack = pVM->hmr0.s.vmx.idLbrFromIpMsrLast - pVM->hmr0.s.vmx.idLbrFromIpMsrFirst + 1;
761	uint32_t const idxMsr = idMsr - pVM->hmr0.s.vmx.idLbrFromIpMsrFirst;
762	if (idxMsr < cLbrStack)
763	{
764	if (pidxMsr)
765	*pidxMsr = idxMsr;
766	return true;
767	}
768	return false;
769	}
770
771
772	/**
773	* Checks if the given MSR is part of the lastbranch-to-IP MSR stack.
774	* @returns @c true if it's part of LBR stack, @c false otherwise.
775	*
776	* @param pVM The cross context VM structure.
777	* @param idMsr The MSR.
778	* @param pidxMsr Where to store the index of the MSR in the LBR MSR array.
779	* Optional, can be NULL.
780	*
781	* @remarks Must only be called when LBR is enabled and when lastbranch-to-IP MSRs
782	* are supported by the CPU (see vmxHCSetupLbrMsrRange).
783	*/
784	DECL_FORCE_INLINE(bool) vmxHCIsLbrBranchToMsr(PCVMCC pVM, uint32_t idMsr, uint32_t *pidxMsr)
785	{
786	Assert(VM_IS_VMX_LBR(pVM));
787	if (pVM->hmr0.s.vmx.idLbrToIpMsrFirst)
788	{
789	uint32_t const cLbrStack = pVM->hmr0.s.vmx.idLbrToIpMsrLast - pVM->hmr0.s.vmx.idLbrToIpMsrFirst + 1;
790	uint32_t const idxMsr = idMsr - pVM->hmr0.s.vmx.idLbrToIpMsrFirst;
791	if (idxMsr < cLbrStack)
792	{
793	if (pidxMsr)
794	*pidxMsr = idxMsr;
795	return true;
796	}
797	}
798	return false;
799	}
800	#endif
801
802
803	/**
804	* Gets the CR0 guest/host mask.
805	*
806	* These bits typically does not change through the lifetime of a VM. Any bit set in
807	* this mask is owned by the host/hypervisor and would cause a VM-exit when modified
808	* by the guest.
809	*
810	* @returns The CR0 guest/host mask.
811	* @param pVCpu The cross context virtual CPU structure.
812	*/
813	static uint64_t vmxHCGetFixedCr0Mask(PCVMCPUCC pVCpu)
814	{
815	/*
816	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
817	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
818	*
819	* Furthermore, modifications to any bits that are reserved/unspecified currently
820	* by the Intel spec. must also cause a VM-exit. This prevents unpredictable behavior
821	* when future CPUs specify and use currently reserved/unspecified bits.
822	*/
823	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
824	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
825	* and @bugref{6944}. */
826	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
827	return ( X86_CR0_PE
828	\| X86_CR0_NE
829	\| (VM_IS_VMX_NESTED_PAGING(pVM) ? 0 : X86_CR0_WP)
830	\| X86_CR0_PG
831	\| VMX_EXIT_HOST_CR0_IGNORE_MASK);
832	}
833
834
835	/**
836	* Gets the CR4 guest/host mask.
837	*
838	* These bits typically does not change through the lifetime of a VM. Any bit set in
839	* this mask is owned by the host/hypervisor and would cause a VM-exit when modified
840	* by the guest.
841	*
842	* @returns The CR4 guest/host mask.
843	* @param pVCpu The cross context virtual CPU structure.
844	*/
845	static uint64_t vmxHCGetFixedCr4Mask(PCVMCPUCC pVCpu)
846	{
847	/*
848	* We construct a mask of all CR4 bits that the guest can modify without causing
849	* a VM-exit. Then invert this mask to obtain all CR4 bits that should cause
850	* a VM-exit when the guest attempts to modify them when executing using
851	* hardware-assisted VMX.
852	*
853	* When a feature is not exposed to the guest (and may be present on the host),
854	* we want to intercept guest modifications to the bit so we can emulate proper
855	* behavior (e.g., #GP).
856	*
857	* Furthermore, only modifications to those bits that don't require immediate
858	* emulation is allowed. For e.g., PCIDE is excluded because the behavior
859	* depends on CR3 which might not always be the guest value while executing
860	* using hardware-assisted VMX.
861	*/
862	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
863	bool const fFsGsBase = pVM->cpum.ro.GuestFeatures.fFsGsBase;
864	bool const fXSaveRstor = pVM->cpum.ro.GuestFeatures.fXSaveRstor;
865	bool const fFxSaveRstor = pVM->cpum.ro.GuestFeatures.fFxSaveRstor;
866
867	/*
868	* Paranoia.
869	* Ensure features exposed to the guest are present on the host.
870	*/
871	Assert(!fFsGsBase \|\| pVM->cpum.ro.HostFeatures.fFsGsBase);
872	Assert(!fXSaveRstor \|\| pVM->cpum.ro.HostFeatures.fXSaveRstor);
873	Assert(!fFxSaveRstor \|\| pVM->cpum.ro.HostFeatures.fFxSaveRstor);
874
875	uint64_t const fGstMask = ( X86_CR4_PVI
876	\| X86_CR4_TSD
877	\| X86_CR4_DE
878	\| X86_CR4_MCE
879	\| X86_CR4_PCE
880	\| X86_CR4_OSXMMEEXCPT
881	\| (fFsGsBase ? X86_CR4_FSGSBASE : 0)
882	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
883	\| (fFxSaveRstor ? X86_CR4_OSFXSR : 0));
884	return ~fGstMask;
885	}
886
887
888	/**
889	* Returns whether the VM-exit MSR-store area differs from the VM-exit MSR-load
890	* area.
891	*
892	* @returns @c true if it's different, @c false otherwise.
893	* @param pVmcsInfo The VMCS info. object.
894	*/
895	DECL_FORCE_INLINE(bool) vmxHCIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
896	{
897	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
898	&& pVmcsInfo->pvGuestMsrStore);
899	}
900
901
902	/**
903	* Sets the given Processor-based VM-execution controls.
904	*
905	* @param pVCpu The cross context virtual CPU structure.
906	* @param pVmxTransient The VMX-transient structure.
907	* @param uProcCtls The Processor-based VM-execution controls to set.
908	*/
909	static void vmxHCSetProcCtlsVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
910	{
911	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
912	if ((pVmcsInfo->u32ProcCtls & uProcCtls) != uProcCtls)
913	{
914	pVmcsInfo->u32ProcCtls \|= uProcCtls;
915	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
916	AssertRC(rc);
917	}
918	}
919
920
921	/**
922	* Removes the given Processor-based VM-execution controls.
923	*
924	* @param pVCpu The cross context virtual CPU structure.
925	* @param pVmxTransient The VMX-transient structure.
926	* @param uProcCtls The Processor-based VM-execution controls to remove.
927	*
928	* @remarks When executing a nested-guest, this will not remove any of the specified
929	* controls if the nested hypervisor has set any one of them.
930	*/
931	static void vmxHCRemoveProcCtlsVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
932	{
933	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
934	if (pVmcsInfo->u32ProcCtls & uProcCtls)
935	{
936	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
937	if ( !pVmxTransient->fIsNestedGuest
938	\|\| !CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, uProcCtls))
939	#else
940	NOREF(pVCpu);
941	if (!pVmxTransient->fIsNestedGuest)
942	#endif
943	{
944	pVmcsInfo->u32ProcCtls &= ~uProcCtls;
945	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
946	AssertRC(rc);
947	}
948	}
949	}
950
951
952	/**
953	* Sets the TSC offset for the current VMCS.
954	*
955	* @param pVCpu The cross context virtual CPU structure.
956	* @param uTscOffset The TSC offset to set.
957	* @param pVmcsInfo The VMCS info. object.
958	*/
959	static void vmxHCSetTscOffsetVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t uTscOffset)
960	{
961	if (pVmcsInfo->u64TscOffset != uTscOffset)
962	{
963	int rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
964	AssertRC(rc);
965	pVmcsInfo->u64TscOffset = uTscOffset;
966	}
967	}
968
969
970	/**
971	* Adds one or more exceptions to the exception bitmap and commits it to the current
972	* VMCS.
973	*
974	* @param pVCpu The cross context virtual CPU structure.
975	* @param pVmxTransient The VMX-transient structure.
976	* @param uXcptMask The exception(s) to add.
977	*/
978	static void vmxHCAddXcptInterceptMask(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
979	{
980	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
981	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
982	if ((uXcptBitmap & uXcptMask) != uXcptMask)
983	{
984	uXcptBitmap \|= uXcptMask;
985	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
986	AssertRC(rc);
987	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
988	}
989	}
990
991
992	/**
993	* Adds an exception to the exception bitmap and commits it to the current VMCS.
994	*
995	* @param pVCpu The cross context virtual CPU structure.
996	* @param pVmxTransient The VMX-transient structure.
997	* @param uXcpt The exception to add.
998	*/
999	static void vmxHCAddXcptIntercept(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1000	{
1001	Assert(uXcpt <= X86_XCPT_LAST);
1002	vmxHCAddXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT_32(uXcpt));
1003	}
1004
1005
1006	/**
1007	* Remove one or more exceptions from the exception bitmap and commits it to the
1008	* current VMCS.
1009	*
1010	* This takes care of not removing the exception intercept if a nested-guest
1011	* requires the exception to be intercepted.
1012	*
1013	* @returns VBox status code.
1014	* @param pVCpu The cross context virtual CPU structure.
1015	* @param pVmxTransient The VMX-transient structure.
1016	* @param uXcptMask The exception(s) to remove.
1017	*/
1018	static int vmxHCRemoveXcptInterceptMask(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1019	{
1020	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1021	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
1022	if (u32XcptBitmap & uXcptMask)
1023	{
1024	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1025	if (!pVmxTransient->fIsNestedGuest)
1026	{ /* likely */ }
1027	else
1028	uXcptMask &= ~pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32XcptBitmap;
1029	#endif
1030	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
1031	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
1032	\| RT_BIT(X86_XCPT_DE)
1033	\| RT_BIT(X86_XCPT_NM)
1034	\| RT_BIT(X86_XCPT_TS)
1035	\| RT_BIT(X86_XCPT_UD)
1036	\| RT_BIT(X86_XCPT_NP)
1037	\| RT_BIT(X86_XCPT_SS)
1038	\| RT_BIT(X86_XCPT_GP)
1039	\| RT_BIT(X86_XCPT_PF)
1040	\| RT_BIT(X86_XCPT_MF));
1041	#elif defined(HMVMX_ALWAYS_TRAP_PF)
1042	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
1043	#endif
1044	if (uXcptMask)
1045	{
1046	/* Validate we are not removing any essential exception intercepts. */
1047	#ifdef IN_RING0
1048	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF)));
1049	#else
1050	Assert(!(uXcptMask & RT_BIT(X86_XCPT_PF)));
1051	#endif
1052	NOREF(pVCpu);
1053	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
1054	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
1055
1056	/* Remove it from the exception bitmap. */
1057	u32XcptBitmap &= ~uXcptMask;
1058
1059	/* Commit and update the cache if necessary. */
1060	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
1061	{
1062	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1063	AssertRC(rc);
1064	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
1065	}
1066	}
1067	}
1068	return VINF_SUCCESS;
1069	}
1070
1071
1072	/**
1073	* Remove an exceptions from the exception bitmap and commits it to the current
1074	* VMCS.
1075	*
1076	* @returns VBox status code.
1077	* @param pVCpu The cross context virtual CPU structure.
1078	* @param pVmxTransient The VMX-transient structure.
1079	* @param uXcpt The exception to remove.
1080	*/
1081	static int vmxHCRemoveXcptIntercept(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1082	{
1083	return vmxHCRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
1084	}
1085
1086
1087	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1088	/**
1089	* Loads the shadow VMCS specified by the VMCS info. object.
1090	*
1091	* @returns VBox status code.
1092	* @param pVmcsInfo The VMCS info. object.
1093	*
1094	* @remarks Can be called with interrupts disabled.
1095	*/
1096	static int vmxHCLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1097	{
1098	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1099	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1100
1101	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs);
1102	if (RT_SUCCESS(rc))
1103	pVmcsInfo->fShadowVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1104	return rc;
1105	}
1106
1107
1108	/**
1109	* Clears the shadow VMCS specified by the VMCS info. object.
1110	*
1111	* @returns VBox status code.
1112	* @param pVmcsInfo The VMCS info. object.
1113	*
1114	* @remarks Can be called with interrupts disabled.
1115	*/
1116	static int vmxHCClearShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1117	{
1118	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1119	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1120
1121	int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs);
1122	if (RT_SUCCESS(rc))
1123	pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1124	return rc;
1125	}
1126
1127
1128	/**
1129	* Switches from and to the specified VMCSes.
1130	*
1131	* @returns VBox status code.
1132	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
1133	* @param pVmcsInfoTo The VMCS info. object we are switching to.
1134	*
1135	* @remarks Called with interrupts disabled.
1136	*/
1137	static int vmxHCSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
1138	{
1139	/*
1140	* Clear the VMCS we are switching out if it has not already been cleared.
1141	* This will sync any CPU internal data back to the VMCS.
1142	*/
1143	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1144	{
1145	int rc = vmxHCClearVmcs(pVmcsInfoFrom);
1146	if (RT_SUCCESS(rc))
1147	{
1148	/*
1149	* The shadow VMCS, if any, would not be active at this point since we
1150	* would have cleared it while importing the virtual hardware-virtualization
1151	* state as part the VMLAUNCH/VMRESUME VM-exit. Hence, there's no need to
1152	* clear the shadow VMCS here, just assert for safety.
1153	*/
1154	Assert(!pVmcsInfoFrom->pvShadowVmcs \|\| pVmcsInfoFrom->fShadowVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR);
1155	}
1156	else
1157	return rc;
1158	}
1159
1160	/*
1161	* Clear the VMCS we are switching to if it has not already been cleared.
1162	* This will initialize the VMCS launch state to "clear" required for loading it.
1163	*
1164	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
1165	*/
1166	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1167	{
1168	int rc = vmxHCClearVmcs(pVmcsInfoTo);
1169	if (RT_SUCCESS(rc))
1170	{ /* likely */ }
1171	else
1172	return rc;
1173	}
1174
1175	/*
1176	* Finally, load the VMCS we are switching to.
1177	*/
1178	return vmxHCLoadVmcs(pVmcsInfoTo);
1179	}
1180
1181
1182	/**
1183	* Switches between the guest VMCS and the nested-guest VMCS as specified by the
1184	* caller.
1185	*
1186	* @returns VBox status code.
1187	* @param pVCpu The cross context virtual CPU structure.
1188	* @param fSwitchToNstGstVmcs Whether to switch to the nested-guest VMCS (pass
1189	* true) or guest VMCS (pass false).
1190	*/
1191	static int vmxHCSwitchToGstOrNstGstVmcs(PVMCPUCC pVCpu, bool fSwitchToNstGstVmcs)
1192	{
1193	/* Ensure we have synced everything from the guest-CPU context to the VMCS before switching. */
1194	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
1195
1196	PVMXVMCSINFO pVmcsInfoFrom;
1197	PVMXVMCSINFO pVmcsInfoTo;
1198	if (fSwitchToNstGstVmcs)
1199	{
1200	pVmcsInfoFrom = &pVCpu->hmr0.s.vmx.VmcsInfo;
1201	pVmcsInfoTo = &pVCpu->hmr0.s.vmx.VmcsInfoNstGst;
1202	}
1203	else
1204	{
1205	pVmcsInfoFrom = &pVCpu->hmr0.s.vmx.VmcsInfoNstGst;
1206	pVmcsInfoTo = &pVCpu->hmr0.s.vmx.VmcsInfo;
1207	}
1208
1209	/*
1210	* Disable interrupts to prevent being preempted while we switch the current VMCS as the
1211	* preemption hook code path acquires the current VMCS.
1212	*/
1213	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1214
1215	int rc = vmxHCSwitchVmcs(pVmcsInfoFrom, pVmcsInfoTo);
1216	if (RT_SUCCESS(rc))
1217	{
1218	pVCpu->hmr0.s.vmx.fSwitchedToNstGstVmcs = fSwitchToNstGstVmcs;
1219	VCPU_2_VMXSTATE(pVCpu).vmx.fSwitchedToNstGstVmcsCopyForRing3 = fSwitchToNstGstVmcs;
1220
1221	/*
1222	* If we are switching to a VMCS that was executed on a different host CPU or was
1223	* never executed before, flag that we need to export the host state before executing
1224	* guest/nested-guest code using hardware-assisted VMX.
1225	*
1226	* This could probably be done in a preemptible context since the preemption hook
1227	* will flag the necessary change in host context. However, since preemption is
1228	* already disabled and to avoid making assumptions about host specific code in
1229	* RTMpCpuId when called with preemption enabled, we'll do this while preemption is
1230	* disabled.
1231	*/
1232	if (pVmcsInfoTo->idHostCpuState == RTMpCpuId())
1233	{ /* likely */ }
1234	else
1235	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE);
1236
1237	ASMSetFlags(fEFlags);
1238
1239	/*
1240	* We use a different VM-exit MSR-store areas for the guest and nested-guest. Hence,
1241	* flag that we need to update the host MSR values there. Even if we decide in the
1242	* future to share the VM-exit MSR-store area page between the guest and nested-guest,
1243	* if its content differs, we would have to update the host MSRs anyway.
1244	*/
1245	pVCpu->hmr0.s.vmx.fUpdatedHostAutoMsrs = false;
1246	}
1247	else
1248	ASMSetFlags(fEFlags);
1249	return rc;
1250	}
1251	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
1252
1253
1254	/**
1255	* Updates the VM's last error record.
1256	*
1257	* If there was a VMX instruction error, reads the error data from the VMCS and
1258	* updates VCPU's last error record as well.
1259	*
1260	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1261	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
1262	* VERR_VMX_INVALID_VMCS_FIELD.
1263	* @param rc The error code.
1264	*/
1265	static void vmxHCUpdateErrorRecord(PVMCPUCC pVCpu, int rc)
1266	{
1267	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
1268	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
1269	{
1270	AssertPtrReturnVoid(pVCpu);
1271	VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_VM_INSTR_ERROR, &VCPU_2_VMXSTATE(pVCpu).vmx.LastError.u32InstrError);
1272	}
1273	#ifdef IN_RING0
1274	pVCpu->CTX_SUFF(pVM)->hm.s.ForR3.rcInit = rc;
1275	#endif
1276	}
1277
1278
1279	#ifdef VBOX_STRICT
1280	/**
1281	* Reads the VM-entry interruption-information field from the VMCS into the VMX
1282	* transient structure.
1283	*
1284	* @param pVCpu The cross context virtual CPU structure.
1285	* @param pVmxTransient The VMX-transient structure.
1286	*/
1287	DECLINLINE(void) vmxHCReadEntryIntInfoVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1288	{
1289	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
1290	AssertRC(rc);
1291	}
1292
1293
1294	/**
1295	* Reads the VM-entry exception error code field from the VMCS into
1296	* the VMX transient structure.
1297	*
1298	* @param pVCpu The cross context virtual CPU structure.
1299	* @param pVmxTransient The VMX-transient structure.
1300	*/
1301	DECLINLINE(void) vmxHCReadEntryXcptErrorCodeVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1302	{
1303	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1304	AssertRC(rc);
1305	}
1306
1307
1308	/**
1309	* Reads the VM-entry exception error code field from the VMCS into
1310	* the VMX transient structure.
1311	*
1312	* @param pVCpu The cross context virtual CPU structure.
1313	* @param pVmxTransient The VMX-transient structure.
1314	*/
1315	DECLINLINE(void) vmxHCReadEntryInstrLenVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1316	{
1317	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1318	AssertRC(rc);
1319	}
1320	#endif /* VBOX_STRICT */
1321
1322
1323	/**
1324	* Reads the VM-exit interruption-information field from the VMCS into the VMX
1325	* transient structure.
1326	*
1327	* @param pVCpu The cross context virtual CPU structure.
1328	* @param pVmxTransient The VMX-transient structure.
1329	*/
1330	DECLINLINE(void) vmxHCReadExitIntInfoVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1331	{
1332	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1333	{
1334	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1335	AssertRC(rc);
1336	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
1337	}
1338	}
1339
1340
1341	/**
1342	* Reads the VM-exit interruption error code from the VMCS into the VMX
1343	* transient structure.
1344	*
1345	* @param pVCpu The cross context virtual CPU structure.
1346	* @param pVmxTransient The VMX-transient structure.
1347	*/
1348	DECLINLINE(void) vmxHCReadExitIntErrorCodeVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1349	{
1350	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1351	{
1352	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1353	AssertRC(rc);
1354	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1355	}
1356	}
1357
1358
1359	/**
1360	* Reads the VM-exit instruction length field from the VMCS into the VMX
1361	* transient structure.
1362	*
1363	* @param pVCpu The cross context virtual CPU structure.
1364	* @param pVmxTransient The VMX-transient structure.
1365	*/
1366	DECLINLINE(void) vmxHCReadExitInstrLenVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1367	{
1368	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1369	{
1370	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1371	AssertRC(rc);
1372	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1373	}
1374	}
1375
1376
1377	/**
1378	* Reads the VM-exit instruction-information field from the VMCS into
1379	* the VMX transient structure.
1380	*
1381	* @param pVCpu The cross context virtual CPU structure.
1382	* @param pVmxTransient The VMX-transient structure.
1383	*/
1384	DECLINLINE(void) vmxHCReadExitInstrInfoVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1385	{
1386	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1387	{
1388	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1389	AssertRC(rc);
1390	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1391	}
1392	}
1393
1394
1395	/**
1396	* Reads the Exit Qualification from the VMCS into the VMX transient structure.
1397	*
1398	* @param pVCpu The cross context virtual CPU structure.
1399	* @param pVmxTransient The VMX-transient structure.
1400	*/
1401	DECLINLINE(void) vmxHCReadExitQualVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1402	{
1403	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1404	{
1405	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1406	AssertRC(rc);
1407	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1408	}
1409	}
1410
1411
1412	/**
1413	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1414	*
1415	* @param pVCpu The cross context virtual CPU structure.
1416	* @param pVmxTransient The VMX-transient structure.
1417	*/
1418	DECLINLINE(void) vmxHCReadGuestLinearAddrVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1419	{
1420	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1421	{
1422	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1423	AssertRC(rc);
1424	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1425	}
1426	}
1427
1428
1429	/**
1430	* Reads the Guest-physical address from the VMCS into the VMX transient structure.
1431	*
1432	* @param pVCpu The cross context virtual CPU structure.
1433	* @param pVmxTransient The VMX-transient structure.
1434	*/
1435	DECLINLINE(void) vmxHCReadGuestPhysicalAddrVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1436	{
1437	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_PHYSICAL_ADDR))
1438	{
1439	int rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1440	AssertRC(rc);
1441	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_PHYSICAL_ADDR;
1442	}
1443	}
1444
1445	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1446	/**
1447	* Reads the Guest pending-debug exceptions from the VMCS into the VMX transient
1448	* structure.
1449	*
1450	* @param pVCpu The cross context virtual CPU structure.
1451	* @param pVmxTransient The VMX-transient structure.
1452	*/
1453	DECLINLINE(void) vmxHCReadGuestPendingDbgXctps(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1454	{
1455	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_PENDING_DBG_XCPTS))
1456	{
1457	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &pVmxTransient->uGuestPendingDbgXcpts);
1458	AssertRC(rc);
1459	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_PENDING_DBG_XCPTS;
1460	}
1461	}
1462	#endif
1463
1464	/**
1465	* Reads the IDT-vectoring information field from the VMCS into the VMX
1466	* transient structure.
1467	*
1468	* @param pVCpu The cross context virtual CPU structure.
1469	* @param pVmxTransient The VMX-transient structure.
1470	*
1471	* @remarks No-long-jump zone!!!
1472	*/
1473	DECLINLINE(void) vmxHCReadIdtVectoringInfoVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1474	{
1475	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1476	{
1477	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1478	AssertRC(rc);
1479	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1480	}
1481	}
1482
1483
1484	/**
1485	* Reads the IDT-vectoring error code from the VMCS into the VMX
1486	* transient structure.
1487	*
1488	* @param pVCpu The cross context virtual CPU structure.
1489	* @param pVmxTransient The VMX-transient structure.
1490	*/
1491	DECLINLINE(void) vmxHCReadIdtVectoringErrorCodeVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1492	{
1493	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1494	{
1495	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1496	AssertRC(rc);
1497	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1498	}
1499	}
1500
1501	#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
1502	/**
1503	* Reads all relevant read-only VMCS fields into the VMX transient structure.
1504	*
1505	* @param pVCpu The cross context virtual CPU structure.
1506	* @param pVmxTransient The VMX-transient structure.
1507	*/
1508	static void vmxHCReadAllRoFieldsVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1509	{
1510	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1511	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1512	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1513	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1514	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1515	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1516	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1517	rc \|= VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1518	rc \|= VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1519	AssertRC(rc);
1520	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION
1521	\| HMVMX_READ_EXIT_INSTR_LEN
1522	\| HMVMX_READ_EXIT_INSTR_INFO
1523	\| HMVMX_READ_IDT_VECTORING_INFO
1524	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
1525	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
1526	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
1527	\| HMVMX_READ_GUEST_LINEAR_ADDR
1528	\| HMVMX_READ_GUEST_PHYSICAL_ADDR;
1529	}
1530	#endif
1531
1532	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1533	/**
1534	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
1535	*
1536	* @returns @c true if the MSR is intercepted, @c false otherwise.
1537	* @param pbMsrBitmap The MSR bitmap.
1538	* @param offMsr The MSR byte offset.
1539	* @param iBit The bit offset from the byte offset.
1540	*/
1541	DECLINLINE(bool) vmxHCIsMsrBitSet(uint8_t const *pbMsrBitmap, uint16_t offMsr, int32_t iBit)
1542	{
1543	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
1544	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
1545	}
1546	#endif
1547
1548	/**
1549	* Sets the permission bits for the specified MSR in the given MSR bitmap.
1550	*
1551	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
1552	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
1553	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
1554	* the read/write access of this MSR.
1555	*
1556	* @param pVCpu The cross context virtual CPU structure.
1557	* @param pVmcsInfo The VMCS info. object.
1558	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1559	* @param idMsr The MSR value.
1560	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
1561	* include both a read -and- a write permission!
1562	*
1563	* @sa CPUMGetVmxMsrPermission.
1564	* @remarks Can be called with interrupts disabled.
1565	*/
1566	static void vmxHCSetMsrPermission(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
1567	{
1568	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
1569	Assert(pbMsrBitmap);
1570	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
1571
1572	/*
1573	* MSR-bitmap Layout:
1574	* Byte index MSR range Interpreted as
1575	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1576	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1577	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1578	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1579	*
1580	* A bit corresponding to an MSR within the above range causes a VM-exit
1581	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
1582	* the MSR range, it always cause a VM-exit.
1583	*
1584	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
1585	*/
1586	uint16_t const offBitmapRead = 0;
1587	uint16_t const offBitmapWrite = 0x800;
1588	uint16_t offMsr;
1589	int32_t iBit;
1590	if (idMsr <= UINT32_C(0x00001fff))
1591	{
1592	offMsr = 0;
1593	iBit = idMsr;
1594	}
1595	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1596	{
1597	offMsr = 0x400;
1598	iBit = idMsr - UINT32_C(0xc0000000);
1599	}
1600	else
1601	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
1602
1603	/*
1604	* Set the MSR read permission.
1605	*/
1606	uint16_t const offMsrRead = offBitmapRead + offMsr;
1607	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
1608	if (fMsrpm & VMXMSRPM_ALLOW_RD)
1609	{
1610	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1611	bool const fClear = !fIsNstGstVmcs ? true
1612	: !vmxHCIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap, offMsrRead, iBit);
1613	#else
1614	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1615	bool const fClear = true;
1616	#endif
1617	if (fClear)
1618	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
1619	}
1620	else
1621	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
1622
1623	/*
1624	* Set the MSR write permission.
1625	*/
1626	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
1627	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
1628	if (fMsrpm & VMXMSRPM_ALLOW_WR)
1629	{
1630	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1631	bool const fClear = !fIsNstGstVmcs ? true
1632	: !vmxHCIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap, offMsrWrite, iBit);
1633	#else
1634	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1635	bool const fClear = true;
1636	#endif
1637	if (fClear)
1638	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
1639	}
1640	else
1641	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
1642	}
1643
1644
1645	/**
1646	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1647	* area.
1648	*
1649	* @returns VBox status code.
1650	* @param pVCpu The cross context virtual CPU structure.
1651	* @param pVmcsInfo The VMCS info. object.
1652	* @param cMsrs The number of MSRs.
1653	*/
1654	static int vmxHCSetAutoLoadStoreMsrCount(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
1655	{
1656	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1657	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(g_HmMsrs.u.vmx.u64Misc);
1658	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
1659	{
1660	/* Commit the MSR counts to the VMCS and update the cache. */
1661	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
1662	{
1663	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRC(rc);
1664	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRC(rc);
1665	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRC(rc);
1666	pVmcsInfo->cEntryMsrLoad = cMsrs;
1667	pVmcsInfo->cExitMsrStore = cMsrs;
1668	pVmcsInfo->cExitMsrLoad = cMsrs;
1669	}
1670	return VINF_SUCCESS;
1671	}
1672
1673	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
1674	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1675	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1676	}
1677
1678
1679	/**
1680	* Adds a new (or updates the value of an existing) guest/host MSR
1681	* pair to be swapped during the world-switch as part of the
1682	* auto-load/store MSR area in the VMCS.
1683	*
1684	* @returns VBox status code.
1685	* @param pVCpu The cross context virtual CPU structure.
1686	* @param pVmxTransient The VMX-transient structure.
1687	* @param idMsr The MSR.
1688	* @param uGuestMsrValue Value of the guest MSR.
1689	* @param fSetReadWrite Whether to set the guest read/write access of this
1690	* MSR (thus not causing a VM-exit).
1691	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1692	* necessary.
1693	*/
1694	static int vmxHCAddAutoLoadStoreMsr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
1695	bool fSetReadWrite, bool fUpdateHostMsr)
1696	{
1697	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1698	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1699	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1700	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1701	uint32_t i;
1702
1703	/* Paranoia. */
1704	Assert(pGuestMsrLoad);
1705
1706	#ifndef DEBUG_bird
1707	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGuestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
1708	#endif
1709
1710	/* Check if the MSR already exists in the VM-entry MSR-load area. */
1711	for (i = 0; i < cMsrs; i++)
1712	{
1713	if (pGuestMsrLoad[i].u32Msr == idMsr)
1714	break;
1715	}
1716
1717	bool fAdded = false;
1718	if (i == cMsrs)
1719	{
1720	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
1721	++cMsrs;
1722	int rc = vmxHCSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1723	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
1724
1725	/* Set the guest to read/write this MSR without causing VM-exits. */
1726	if ( fSetReadWrite
1727	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
1728	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
1729
1730	Log4Func(("Added MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
1731	fAdded = true;
1732	}
1733
1734	/* Update the MSR value for the newly added or already existing MSR. */
1735	pGuestMsrLoad[i].u32Msr = idMsr;
1736	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
1737
1738	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
1739	if (vmxHCIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1740	{
1741	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1742	pGuestMsrStore[i].u32Msr = idMsr;
1743	pGuestMsrStore[i].u64Value = uGuestMsrValue;
1744	}
1745
1746	/* Update the corresponding slot in the host MSR area. */
1747	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1748	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
1749	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
1750	pHostMsr[i].u32Msr = idMsr;
1751
1752	#ifdef IN_RING0
1753	/*
1754	* Only if the caller requests to update the host MSR value AND we've newly added the
1755	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
1756	* updated by vmxHCUpdateAutoLoadHostMsrs().
1757	*
1758	* We do this for performance reasons since reading MSRs may be quite expensive.
1759	*/
1760	if (fAdded)
1761	{
1762	if (fUpdateHostMsr)
1763	{
1764	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1765	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1766	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
1767	}
1768	else
1769	{
1770	/* Someone else can do the work. */
1771	pVCpu->hmr0.s.vmx.fUpdatedHostAutoMsrs = false;
1772	}
1773	}
1774	#else
1775	RT_NOREF(fUpdateHostMsr);
1776	#endif
1777	return VINF_SUCCESS;
1778	}
1779
1780
1781	/**
1782	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1783	* auto-load/store MSR area in the VMCS.
1784	*
1785	* @returns VBox status code.
1786	* @param pVCpu The cross context virtual CPU structure.
1787	* @param pVmxTransient The VMX-transient structure.
1788	* @param idMsr The MSR.
1789	*/
1790	static int vmxHCRemoveAutoLoadStoreMsr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t idMsr)
1791	{
1792	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1793	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1794	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1795	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1796
1797	#ifndef DEBUG_bird
1798	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
1799	#endif
1800
1801	for (uint32_t i = 0; i < cMsrs; i++)
1802	{
1803	/* Find the MSR. */
1804	if (pGuestMsrLoad[i].u32Msr == idMsr)
1805	{
1806	/*
1807	* If it's the last MSR, we only need to reduce the MSR count.
1808	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
1809	*/
1810	if (i < cMsrs - 1)
1811	{
1812	/* Remove it from the VM-entry MSR-load area. */
1813	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
1814	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
1815
1816	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
1817	if (vmxHCIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1818	{
1819	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1820	Assert(pGuestMsrStore[i].u32Msr == idMsr);
1821	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
1822	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
1823	}
1824
1825	/* Remove it from the VM-exit MSR-load area. */
1826	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1827	Assert(pHostMsr[i].u32Msr == idMsr);
1828	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
1829	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
1830	}
1831
1832	/* Reduce the count to reflect the removed MSR and bail. */
1833	--cMsrs;
1834	break;
1835	}
1836	}
1837
1838	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
1839	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
1840	{
1841	int rc = vmxHCSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1842	AssertRCReturn(rc, rc);
1843
1844	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1845	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1846	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
1847
1848	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
1849	return VINF_SUCCESS;
1850	}
1851
1852	return VERR_NOT_FOUND;
1853	}
1854
1855
1856	/**
1857	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
1858	*
1859	* @returns @c true if found, @c false otherwise.
1860	* @param pVmcsInfo The VMCS info. object.
1861	* @param idMsr The MSR to find.
1862	*/
1863	static bool vmxHCIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
1864	{
1865	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1866	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
1867	Assert(pMsrs);
1868	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
1869	for (uint32_t i = 0; i < cMsrs; i++)
1870	{
1871	if (pMsrs[i].u32Msr == idMsr)
1872	return true;
1873	}
1874	return false;
1875	}
1876
1877
1878	/**
1879	* Verifies that our cached values of the VMCS fields are all consistent with
1880	* what's actually present in the VMCS.
1881	*
1882	* @returns VBox status code.
1883	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
1884	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
1885	* VMCS content. HMCPU error-field is
1886	* updated, see VMX_VCI_XXX.
1887	* @param pVCpu The cross context virtual CPU structure.
1888	* @param pVmcsInfo The VMCS info. object.
1889	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1890	*/
1891	static int vmxHCCheckCachedVmcsCtls(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1892	{
1893	const char * const pcszVmcs = fIsNstGstVmcs ? "Nested-guest VMCS" : "VMCS";
1894
1895	uint32_t u32Val;
1896	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY, &u32Val);
1897	AssertRC(rc);
1898	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
1899	("%s entry controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32EntryCtls, u32Val),
1900	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_ENTRY,
1901	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1902
1903	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_EXIT, &u32Val);
1904	AssertRC(rc);
1905	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
1906	("%s exit controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ExitCtls, u32Val),
1907	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_EXIT,
1908	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1909
1910	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1911	AssertRC(rc);
1912	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
1913	("%s pin controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32PinCtls, u32Val),
1914	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PIN_EXEC,
1915	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1916
1917	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1918	AssertRC(rc);
1919	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
1920	("%s proc controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls, u32Val),
1921	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PROC_EXEC,
1922	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1923
1924	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
1925	{
1926	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1927	AssertRC(rc);
1928	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
1929	("%s proc2 controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls2, u32Val),
1930	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
1931	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1932	}
1933
1934	uint64_t u64Val;
1935	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TERTIARY_CTLS)
1936	{
1937	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_CTRL_PROC_EXEC3_FULL, &u64Val);
1938	AssertRC(rc);
1939	AssertMsgReturnStmt(pVmcsInfo->u64ProcCtls3 == u64Val,
1940	("%s proc3 controls mismatch: Cache=%#RX32 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64ProcCtls3, u64Val),
1941	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PROC_EXEC3,
1942	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1943	}
1944
1945	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
1946	AssertRC(rc);
1947	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
1948	("%s exception bitmap mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32XcptBitmap, u32Val),
1949	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
1950	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1951
1952	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
1953	AssertRC(rc);
1954	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
1955	("%s TSC offset mismatch: Cache=%#RX64 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64TscOffset, u64Val),
1956	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
1957	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1958
1959	NOREF(pcszVmcs);
1960	return VINF_SUCCESS;
1961	}
1962
1963
1964	#ifdef IN_RING0
1965	/**
1966	* Sets up the LBR MSR ranges based on the host CPU.
1967	*
1968	* @returns VBox status code.
1969	* @param pVM The cross context VM structure.
1970	*/
1971	static int vmxHCSetupLbrMsrRange(PVMCC pVM)
1972	{
1973	Assert(VM_IS_VMX_LBR(pVM));
1974	uint32_t idLbrFromIpMsrFirst;
1975	uint32_t idLbrFromIpMsrLast;
1976	uint32_t idLbrToIpMsrFirst;
1977	uint32_t idLbrToIpMsrLast;
1978	uint32_t idLbrTosMsr;
1979
1980	/*
1981	* Determine the LBR MSRs supported for this host CPU family and model.
1982	*
1983	* See Intel spec. 17.4.8 "LBR Stack".
1984	* See Intel "Model-Specific Registers" spec.
1985	*/
1986	uint32_t const uFamilyModel = (pVM->cpum.ro.HostFeatures.uFamily << 8)
1987	\| pVM->cpum.ro.HostFeatures.uModel;
1988	switch (uFamilyModel)
1989	{
1990	case 0x0f01: case 0x0f02:
1991	idLbrFromIpMsrFirst = MSR_P4_LASTBRANCH_0;
1992	idLbrFromIpMsrLast = MSR_P4_LASTBRANCH_3;
1993	idLbrToIpMsrFirst = 0x0;
1994	idLbrToIpMsrLast = 0x0;
1995	idLbrTosMsr = MSR_P4_LASTBRANCH_TOS;
1996	break;
1997
1998	case 0x065c: case 0x065f: case 0x064e: case 0x065e: case 0x068e:
1999	case 0x069e: case 0x0655: case 0x0666: case 0x067a: case 0x0667:
2000	case 0x066a: case 0x066c: case 0x067d: case 0x067e:
2001	idLbrFromIpMsrFirst = MSR_LASTBRANCH_0_FROM_IP;
2002	idLbrFromIpMsrLast = MSR_LASTBRANCH_31_FROM_IP;
2003	idLbrToIpMsrFirst = MSR_LASTBRANCH_0_TO_IP;
2004	idLbrToIpMsrLast = MSR_LASTBRANCH_31_TO_IP;
2005	idLbrTosMsr = MSR_LASTBRANCH_TOS;
2006	break;
2007
2008	case 0x063d: case 0x0647: case 0x064f: case 0x0656: case 0x063c:
2009	case 0x0645: case 0x0646: case 0x063f: case 0x062a: case 0x062d:
2010	case 0x063a: case 0x063e: case 0x061a: case 0x061e: case 0x061f:
2011	case 0x062e: case 0x0625: case 0x062c: case 0x062f:
2012	idLbrFromIpMsrFirst = MSR_LASTBRANCH_0_FROM_IP;
2013	idLbrFromIpMsrLast = MSR_LASTBRANCH_15_FROM_IP;
2014	idLbrToIpMsrFirst = MSR_LASTBRANCH_0_TO_IP;
2015	idLbrToIpMsrLast = MSR_LASTBRANCH_15_TO_IP;
2016	idLbrTosMsr = MSR_LASTBRANCH_TOS;
2017	break;
2018
2019	case 0x0617: case 0x061d: case 0x060f:
2020	idLbrFromIpMsrFirst = MSR_CORE2_LASTBRANCH_0_FROM_IP;
2021	idLbrFromIpMsrLast = MSR_CORE2_LASTBRANCH_3_FROM_IP;
2022	idLbrToIpMsrFirst = MSR_CORE2_LASTBRANCH_0_TO_IP;
2023	idLbrToIpMsrLast = MSR_CORE2_LASTBRANCH_3_TO_IP;
2024	idLbrTosMsr = MSR_CORE2_LASTBRANCH_TOS;
2025	break;
2026
2027	/* Atom and related microarchitectures we don't care about:
2028	case 0x0637: case 0x064a: case 0x064c: case 0x064d: case 0x065a:
2029	case 0x065d: case 0x061c: case 0x0626: case 0x0627: case 0x0635:
2030	case 0x0636: */
2031	/* All other CPUs: */
2032	default:
2033	{
2034	LogRelFunc(("Could not determine LBR stack size for the CPU model %#x\n", uFamilyModel));
2035	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_LBR_STACK_SIZE_UNKNOWN;
2036	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2037	}
2038	}
2039
2040	/*
2041	* Validate.
2042	*/
2043	uint32_t const cLbrStack = idLbrFromIpMsrLast - idLbrFromIpMsrFirst + 1;
2044	PCVMCPU pVCpu0 = VMCC_GET_CPU_0(pVM);
2045	AssertCompile( RT_ELEMENTS(pVCpu0->hm.s.vmx.VmcsInfo.au64LbrFromIpMsr)
2046	== RT_ELEMENTS(pVCpu0->hm.s.vmx.VmcsInfo.au64LbrToIpMsr));
2047	if (cLbrStack > RT_ELEMENTS(pVCpu0->hm.s.vmx.VmcsInfo.au64LbrFromIpMsr))
2048	{
2049	LogRelFunc(("LBR stack size of the CPU (%u) exceeds our buffer size\n", cLbrStack));
2050	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_LBR_STACK_SIZE_OVERFLOW;
2051	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2052	}
2053	NOREF(pVCpu0);
2054
2055	/*
2056	* Update the LBR info. to the VM struct. for use later.
2057	*/
2058	pVM->hmr0.s.vmx.idLbrTosMsr = idLbrTosMsr;
2059
2060	pVM->hm.s.ForR3.vmx.idLbrFromIpMsrFirst = pVM->hmr0.s.vmx.idLbrFromIpMsrFirst = idLbrFromIpMsrFirst;
2061	pVM->hm.s.ForR3.vmx.idLbrFromIpMsrLast = pVM->hmr0.s.vmx.idLbrFromIpMsrLast = idLbrFromIpMsrLast;
2062
2063	pVM->hm.s.ForR3.vmx.idLbrToIpMsrFirst = pVM->hmr0.s.vmx.idLbrToIpMsrFirst = idLbrToIpMsrFirst;
2064	pVM->hm.s.ForR3.vmx.idLbrToIpMsrLast = pVM->hmr0.s.vmx.idLbrToIpMsrLast = idLbrToIpMsrLast;
2065	return VINF_SUCCESS;
2066	}
2067
2068
2069	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2070	/**
2071	* Sets up the shadow VMCS fields arrays.
2072	*
2073	* This function builds arrays of VMCS fields to sync the shadow VMCS later while
2074	* executing the guest.
2075	*
2076	* @returns VBox status code.
2077	* @param pVM The cross context VM structure.
2078	*/
2079	static int vmxHCSetupShadowVmcsFieldsArrays(PVMCC pVM)
2080	{
2081	/*
2082	* Paranoia. Ensure we haven't exposed the VMWRITE-All VMX feature to the guest
2083	* when the host does not support it.
2084	*/
2085	bool const fGstVmwriteAll = pVM->cpum.ro.GuestFeatures.fVmxVmwriteAll;
2086	if ( !fGstVmwriteAll
2087	\|\| (g_HmMsrs.u.vmx.u64Misc & VMX_MISC_VMWRITE_ALL))
2088	{ /* likely. */ }
2089	else
2090	{
2091	LogRelFunc(("VMX VMWRITE-All feature exposed to the guest but host CPU does not support it!\n"));
2092	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_GST_HOST_VMWRITE_ALL;
2093	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2094	}
2095
2096	uint32_t const cVmcsFields = RT_ELEMENTS(g_aVmcsFields);
2097	uint32_t cRwFields = 0;
2098	uint32_t cRoFields = 0;
2099	for (uint32_t i = 0; i < cVmcsFields; i++)
2100	{
2101	VMXVMCSFIELD VmcsField;
2102	VmcsField.u = g_aVmcsFields[i];
2103
2104	/*
2105	* We will be writing "FULL" (64-bit) fields while syncing the shadow VMCS.
2106	* Therefore, "HIGH" (32-bit portion of 64-bit) fields must not be included
2107	* in the shadow VMCS fields array as they would be redundant.
2108	*
2109	* If the VMCS field depends on a CPU feature that is not exposed to the guest,
2110	* we must not include it in the shadow VMCS fields array. Guests attempting to
2111	* VMREAD/VMWRITE such VMCS fields would cause a VM-exit and we shall emulate
2112	* the required behavior.
2113	*/
2114	if ( VmcsField.n.fAccessType == VMX_VMCSFIELD_ACCESS_FULL
2115	&& CPUMIsGuestVmxVmcsFieldValid(pVM, VmcsField.u))
2116	{
2117	/*
2118	* Read-only fields are placed in a separate array so that while syncing shadow
2119	* VMCS fields later (which is more performance critical) we can avoid branches.
2120	*
2121	* However, if the guest can write to all fields (including read-only fields),
2122	* we treat it a as read/write field. Otherwise, writing to these fields would
2123	* cause a VMWRITE instruction error while syncing the shadow VMCS.
2124	*/
2125	if ( fGstVmwriteAll
2126	\|\| !VMXIsVmcsFieldReadOnly(VmcsField.u))
2127	pVM->hmr0.s.vmx.paShadowVmcsFields[cRwFields++] = VmcsField.u;
2128	else
2129	pVM->hmr0.s.vmx.paShadowVmcsRoFields[cRoFields++] = VmcsField.u;
2130	}
2131	}
2132
2133	/* Update the counts. */
2134	pVM->hmr0.s.vmx.cShadowVmcsFields = cRwFields;
2135	pVM->hmr0.s.vmx.cShadowVmcsRoFields = cRoFields;
2136	return VINF_SUCCESS;
2137	}
2138
2139
2140	/**
2141	* Sets up the VMREAD and VMWRITE bitmaps.
2142	*
2143	* @param pVM The cross context VM structure.
2144	*/
2145	static void vmxHCSetupVmreadVmwriteBitmaps(PVMCC pVM)
2146	{
2147	/*
2148	* By default, ensure guest attempts to access any VMCS fields cause VM-exits.
2149	*/
2150	uint32_t const cbBitmap = X86_PAGE_4K_SIZE;
2151	uint8_t pbVmreadBitmap = (uint8_t )pVM->hmr0.s.vmx.pvVmreadBitmap;
2152	uint8_t pbVmwriteBitmap = (uint8_t )pVM->hmr0.s.vmx.pvVmwriteBitmap;
2153	ASMMemFill32(pbVmreadBitmap, cbBitmap, UINT32_C(0xffffffff));
2154	ASMMemFill32(pbVmwriteBitmap, cbBitmap, UINT32_C(0xffffffff));
2155
2156	/*
2157	* Skip intercepting VMREAD/VMWRITE to guest read/write fields in the
2158	* VMREAD and VMWRITE bitmaps.
2159	*/
2160	{
2161	uint32_t const *paShadowVmcsFields = pVM->hmr0.s.vmx.paShadowVmcsFields;
2162	uint32_t const cShadowVmcsFields = pVM->hmr0.s.vmx.cShadowVmcsFields;
2163	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
2164	{
2165	uint32_t const uVmcsField = paShadowVmcsFields[i];
2166	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
2167	Assert(uVmcsField >> 3 < cbBitmap);
2168	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
2169	ASMBitClear(pbVmwriteBitmap + (uVmcsField >> 3), uVmcsField & 7);
2170	}
2171	}
2172
2173	/*
2174	* Skip intercepting VMREAD for guest read-only fields in the VMREAD bitmap
2175	* if the host supports VMWRITE to all supported VMCS fields.
2176	*/
2177	if (g_HmMsrs.u.vmx.u64Misc & VMX_MISC_VMWRITE_ALL)
2178	{
2179	uint32_t const *paShadowVmcsRoFields = pVM->hmr0.s.vmx.paShadowVmcsRoFields;
2180	uint32_t const cShadowVmcsRoFields = pVM->hmr0.s.vmx.cShadowVmcsRoFields;
2181	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
2182	{
2183	uint32_t const uVmcsField = paShadowVmcsRoFields[i];
2184	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
2185	Assert(uVmcsField >> 3 < cbBitmap);
2186	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
2187	}
2188	}
2189	}
2190	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
2191
2192
2193	/**
2194	* Sets up the APIC-access page address for the VMCS.
2195	*
2196	* @param pVCpu The cross context virtual CPU structure.
2197	*/
2198	DECLINLINE(void) vmxHCSetupVmcsApicAccessAddr(PVMCPUCC pVCpu)
2199	{
2200	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.HCPhysApicAccess;
2201	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
2202	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2203	int rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
2204	AssertRC(rc);
2205	}
2206
2207	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2208
2209	/**
2210	* Sets up the VMREAD bitmap address for the VMCS.
2211	*
2212	* @param pVCpu The cross context virtual CPU structure.
2213	*/
2214	DECLINLINE(void) vmxHCSetupVmcsVmreadBitmapAddr(PVMCPUCC pVCpu)
2215	{
2216	RTHCPHYS const HCPhysVmreadBitmap = pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.HCPhysVmreadBitmap;
2217	Assert(HCPhysVmreadBitmap != NIL_RTHCPHYS);
2218	Assert(!(HCPhysVmreadBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2219	int rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL, HCPhysVmreadBitmap);
2220	AssertRC(rc);
2221	}
2222
2223
2224	/**
2225	* Sets up the VMWRITE bitmap address for the VMCS.
2226	*
2227	* @param pVCpu The cross context virtual CPU structure.
2228	*/
2229	DECLINLINE(void) vmxHCSetupVmcsVmwriteBitmapAddr(PVMCPUCC pVCpu)
2230	{
2231	RTHCPHYS const HCPhysVmwriteBitmap = pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.HCPhysVmwriteBitmap;
2232	Assert(HCPhysVmwriteBitmap != NIL_RTHCPHYS);
2233	Assert(!(HCPhysVmwriteBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2234	int rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL, HCPhysVmwriteBitmap);
2235	AssertRC(rc);
2236	}
2237
2238	#endif
2239
2240	/**
2241	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
2242	*
2243	* @param pVCpu The cross context virtual CPU structure.
2244	* @param pVmcsInfo The VMCS info. object.
2245	*/
2246	static void vmxHCSetupVmcsMsrPermissions(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2247	{
2248	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
2249
2250	/*
2251	* By default, ensure guest attempts to access any MSR cause VM-exits.
2252	* This shall later be relaxed for specific MSRs as necessary.
2253	*
2254	* Note: For nested-guests, the entire bitmap will be merged prior to
2255	* executing the nested-guest using hardware-assisted VMX and hence there
2256	* is no need to perform this operation. See vmxHCMergeMsrBitmapNested.
2257	*/
2258	Assert(pVmcsInfo->pvMsrBitmap);
2259	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
2260
2261	/*
2262	* The guest can access the following MSRs (read, write) without causing
2263	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
2264	*/
2265	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2266	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
2267	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
2268	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
2269	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2270	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
2271
2272	/*
2273	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
2274	* associated with then. We never need to intercept access (writes need to be
2275	* executed without causing a VM-exit, reads will #GP fault anyway).
2276	*
2277	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
2278	* read/write them. We swap the guest/host MSR value using the
2279	* auto-load/store MSR area.
2280	*/
2281	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2282	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
2283	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
2284	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
2285	if (pVM->cpum.ro.GuestFeatures.fIbrs)
2286	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
2287
2288	/*
2289	* Allow full read/write access for the following MSRs (mandatory for VT-x)
2290	* required for 64-bit guests.
2291	*/
2292	if (pVM->hmr0.s.fAllow64BitGuests)
2293	{
2294	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
2295	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
2296	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
2297	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2298	}
2299
2300	/*
2301	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
2302	*/
2303	#ifdef VBOX_STRICT
2304	Assert(pVmcsInfo->pvMsrBitmap);
2305	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
2306	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
2307	#endif
2308	}
2309
2310
2311	/**
2312	* Sets up pin-based VM-execution controls in the VMCS.
2313	*
2314	* @returns VBox status code.
2315	* @param pVCpu The cross context virtual CPU structure.
2316	* @param pVmcsInfo The VMCS info. object.
2317	*/
2318	static int vmxHCSetupVmcsPinCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2319	{
2320	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2321	uint32_t fVal = g_HmMsrs.u.vmx.PinCtls.n.allowed0; /* Bits set here must always be set. */
2322	uint32_t const fZap = g_HmMsrs.u.vmx.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2323
2324	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2325	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2326
2327	if (g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2328	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2329
2330	/* Enable the VMX-preemption timer. */
2331	if (VM_IS_VMX_PREEMPT_TIMER_USED(pVM))
2332	{
2333	Assert(g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2334	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
2335	}
2336
2337	#if 0
2338	/* Enable posted-interrupt processing. */
2339	if (pVM->hm.s.fPostedIntrs)
2340	{
2341	Assert(g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2342	Assert(g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2343	fVal \|= VMX_PIN_CTLS_POSTED_INT;
2344	}
2345	#endif
2346
2347	if ((fVal & fZap) != fVal)
2348	{
2349	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2350	g_HmMsrs.u.vmx.PinCtls.n.allowed0, fVal, fZap));
2351	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2352	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2353	}
2354
2355	/* Commit it to the VMCS and update our cache. */
2356	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2357	AssertRC(rc);
2358	pVmcsInfo->u32PinCtls = fVal;
2359
2360	return VINF_SUCCESS;
2361	}
2362
2363
2364	/**
2365	* Sets up secondary processor-based VM-execution controls in the VMCS.
2366	*
2367	* @returns VBox status code.
2368	* @param pVCpu The cross context virtual CPU structure.
2369	* @param pVmcsInfo The VMCS info. object.
2370	*/
2371	static int vmxHCSetupVmcsProcCtls2(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2372	{
2373	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2374	uint32_t fVal = g_HmMsrs.u.vmx.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
2375	uint32_t const fZap = g_HmMsrs.u.vmx.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2376
2377	/* WBINVD causes a VM-exit. */
2378	if (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2379	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
2380
2381	/* Enable EPT (aka nested-paging). */
2382	if (VM_IS_VMX_NESTED_PAGING(pVM))
2383	fVal \|= VMX_PROC_CTLS2_EPT;
2384
2385	/* Enable the INVPCID instruction if we expose it to the guest and is supported
2386	by the hardware. Without this, guest executing INVPCID would cause a #UD. */
2387	if ( pVM->cpum.ro.GuestFeatures.fInvpcid
2388	&& (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID))
2389	fVal \|= VMX_PROC_CTLS2_INVPCID;
2390
2391	/* Enable VPID. */
2392	if (pVM->hmr0.s.vmx.fVpid)
2393	fVal \|= VMX_PROC_CTLS2_VPID;
2394
2395	/* Enable unrestricted guest execution. */
2396	if (VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
2397	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
2398
2399	#if 0
2400	if (pVM->hm.s.fVirtApicRegs)
2401	{
2402	/* Enable APIC-register virtualization. */
2403	Assert(g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
2404	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
2405
2406	/* Enable virtual-interrupt delivery. */
2407	Assert(g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
2408	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
2409	}
2410	#endif
2411
2412	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is
2413	where the TPR shadow resides. */
2414	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2415	* done dynamically. */
2416	if (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
2417	{
2418	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
2419	vmxHCSetupVmcsApicAccessAddr(pVCpu);
2420	}
2421
2422	/* Enable the RDTSCP instruction if we expose it to the guest and is supported
2423	by the hardware. Without this, guest executing RDTSCP would cause a #UD. */
2424	if ( pVM->cpum.ro.GuestFeatures.fRdTscP
2425	&& (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP))
2426	fVal \|= VMX_PROC_CTLS2_RDTSCP;
2427
2428	/* Enable Pause-Loop exiting. */
2429	if ( (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
2430	&& pVM->hm.s.vmx.cPleGapTicks
2431	&& pVM->hm.s.vmx.cPleWindowTicks)
2432	{
2433	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
2434
2435	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks); AssertRC(rc);
2436	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks); AssertRC(rc);
2437	}
2438
2439	if ((fVal & fZap) != fVal)
2440	{
2441	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2442	g_HmMsrs.u.vmx.ProcCtls2.n.allowed0, fVal, fZap));
2443	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2444	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2445	}
2446
2447	/* Commit it to the VMCS and update our cache. */
2448	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2449	AssertRC(rc);
2450	pVmcsInfo->u32ProcCtls2 = fVal;
2451
2452	return VINF_SUCCESS;
2453	}
2454
2455
2456	/**
2457	* Sets up processor-based VM-execution controls in the VMCS.
2458	*
2459	* @returns VBox status code.
2460	* @param pVCpu The cross context virtual CPU structure.
2461	* @param pVmcsInfo The VMCS info. object.
2462	*/
2463	static int vmxHCSetupVmcsProcCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2464	{
2465	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2466	uint32_t fVal = g_HmMsrs.u.vmx.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
2467	uint32_t const fZap = g_HmMsrs.u.vmx.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2468
2469	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
2470	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2471	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2472	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2473	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2474	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2475	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2476
2477	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2478	if ( !(g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
2479	\|\| (g_HmMsrs.u.vmx.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
2480	{
2481	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2482	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2483	}
2484
2485	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2486	if (!VM_IS_VMX_NESTED_PAGING(pVM))
2487	{
2488	Assert(!VM_IS_VMX_UNRESTRICTED_GUEST(pVM));
2489	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
2490	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
2491	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
2492	}
2493
2494	#ifdef IN_INRG0
2495	/* Use TPR shadowing if supported by the CPU. */
2496	if ( PDMHasApic(pVM)
2497	&& (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
2498	{
2499	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2500	/* CR8 writes cause a VM-exit based on TPR threshold. */
2501	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
2502	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
2503	vmxHCSetupVmcsVirtApicAddr(pVmcsInfo);
2504	}
2505	else
2506	{
2507	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
2508	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
2509	if (pVM->hmr0.s.fAllow64BitGuests)
2510	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2511	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2512	}
2513
2514	/* Use MSR-bitmaps if supported by the CPU. */
2515	if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2516	{
2517	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
2518	vmxHCSetupVmcsMsrBitmapAddr(pVmcsInfo);
2519	}
2520	#endif
2521
2522	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2523	if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2524	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
2525
2526	if ((fVal & fZap) != fVal)
2527	{
2528	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2529	g_HmMsrs.u.vmx.ProcCtls.n.allowed0, fVal, fZap));
2530	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2531	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2532	}
2533
2534	/* Commit it to the VMCS and update our cache. */
2535	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2536	AssertRC(rc);
2537	pVmcsInfo->u32ProcCtls = fVal;
2538
2539	/* Set up MSR permissions that don't change through the lifetime of the VM. */
2540	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2541	vmxHCSetupVmcsMsrPermissions(pVCpu, pVmcsInfo);
2542
2543	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2544	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2545	return vmxHCSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
2546
2547	/* Sanity check, should not really happen. */
2548	if (RT_LIKELY(!VM_IS_VMX_UNRESTRICTED_GUEST(pVM)))
2549	{ /* likely */ }
2550	else
2551	{
2552	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_INVALID_UX_COMBO;
2553	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2554	}
2555
2556	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2557	return VINF_SUCCESS;
2558	}
2559
2560
2561	/**
2562	* Sets up miscellaneous (everything other than Pin, Processor and secondary
2563	* Processor-based VM-execution) control fields in the VMCS.
2564	*
2565	* @returns VBox status code.
2566	* @param pVCpu The cross context virtual CPU structure.
2567	* @param pVmcsInfo The VMCS info. object.
2568	*/
2569	static int vmxHCSetupVmcsMiscCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2570	{
2571	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2572	if (pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fUseVmcsShadowing)
2573	{
2574	vmxHCSetupVmcsVmreadBitmapAddr(pVCpu);
2575	vmxHCSetupVmcsVmwriteBitmapAddr(pVCpu);
2576	}
2577	#endif
2578
2579	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
2580	int rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
2581	AssertRC(rc);
2582
2583	rc = vmxHCSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
2584	if (RT_SUCCESS(rc))
2585	{
2586	uint64_t const u64Cr0Mask = vmxHCGetFixedCr0Mask(pVCpu);
2587	uint64_t const u64Cr4Mask = vmxHCGetFixedCr4Mask(pVCpu);
2588
2589	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask); AssertRC(rc);
2590	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask); AssertRC(rc);
2591
2592	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
2593	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
2594
2595	if (pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fLbr)
2596	{
2597	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS64_GUEST_DEBUGCTL_FULL, MSR_IA32_DEBUGCTL_LBR);
2598	AssertRC(rc);
2599	}
2600	return VINF_SUCCESS;
2601	}
2602	else
2603	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
2604	return rc;
2605	}
2606
2607
2608	/**
2609	* Sets up the initial exception bitmap in the VMCS based on static conditions.
2610	*
2611	* We shall setup those exception intercepts that don't change during the
2612	* lifetime of the VM here. The rest are done dynamically while loading the
2613	* guest state.
2614	*
2615	* @param pVCpu The cross context virtual CPU structure.
2616	* @param pVmcsInfo The VMCS info. object.
2617	*/
2618	static void vmxHCSetupVmcsXcptBitmap(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2619	{
2620	/*
2621	* The following exceptions are always intercepted:
2622	*
2623	* #AC - To prevent the guest from hanging the CPU and for dealing with
2624	* split-lock detecting host configs.
2625	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
2626	* recursive #DBs can cause a CPU hang.
2627	* #PF - To sync our shadow page tables when nested-paging is not used.
2628	*/
2629	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging;
2630	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
2631	\| RT_BIT(X86_XCPT_DB)
2632	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
2633
2634	/* Commit it to the VMCS. */
2635	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2636	AssertRC(rc);
2637
2638	/* Update our cache of the exception bitmap. */
2639	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
2640	}
2641
2642
2643	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2644	/**
2645	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
2646	*
2647	* @returns VBox status code.
2648	* @param pVmcsInfo The VMCS info. object.
2649	*/
2650	static int vmxHCSetupVmcsCtlsNested(PVMXVMCSINFO pVmcsInfo)
2651	{
2652	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
2653	int rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
2654	AssertRC(rc);
2655
2656	rc = vmxHCSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
2657	if (RT_SUCCESS(rc))
2658	{
2659	if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2660	vmxHCSetupVmcsMsrBitmapAddr(pVmcsInfo);
2661
2662	/* Paranoia - We've not yet initialized these, they shall be done while merging the VMCS. */
2663	Assert(!pVmcsInfo->u64Cr0Mask);
2664	Assert(!pVmcsInfo->u64Cr4Mask);
2665	return VINF_SUCCESS;
2666	}
2667	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
2668	return rc;
2669	}
2670	#endif
2671	#endif /* !IN_RING0 */
2672
2673
2674	/**
2675	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
2676	* VMCS.
2677	*
2678	* This is typically required when the guest changes paging mode.
2679	*
2680	* @returns VBox status code.
2681	* @param pVCpu The cross context virtual CPU structure.
2682	* @param pVmxTransient The VMX-transient structure.
2683	*
2684	* @remarks Requires EFER.
2685	* @remarks No-long-jump zone!!!
2686	*/
2687	static int vmxHCExportGuestEntryExitCtls(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
2688	{
2689	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
2690	{
2691	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2692	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2693
2694	/*
2695	* VM-entry controls.
2696	*/
2697	{
2698	uint32_t fVal = g_HmMsrs.u.vmx.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
2699	uint32_t const fZap = g_HmMsrs.u.vmx.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2700
2701	/*
2702	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
2703	* The first VT-x capable CPUs only supported the 1-setting of this bit.
2704	*
2705	* For nested-guests, this is a mandatory VM-entry control. It's also
2706	* required because we do not want to leak host bits to the nested-guest.
2707	*/
2708	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
2709
2710	/*
2711	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
2712	*
2713	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
2714	* required to get the nested-guest working with hardware-assisted VMX execution.
2715	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested hypervisor
2716	* can skip intercepting changes to the EFER MSR. This is why it needs to be done
2717	* here rather than while merging the guest VMCS controls.
2718	*/
2719	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
2720	{
2721	Assert(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME);
2722	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
2723	}
2724	else
2725	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
2726
2727	/*
2728	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
2729	*
2730	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
2731	* regardless of whether the nested-guest VMCS specifies it because we are free to
2732	* load whatever MSRs we require and we do not need to modify the guest visible copy
2733	* of the VM-entry MSR load area.
2734	*/
2735	if ( g_fHmVmxSupportsVmcsEfer
2736	&& vmxHCShouldSwapEferMsr(pVCpu, pVmxTransient))
2737	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
2738	else
2739	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
2740
2741	/*
2742	* The following should -not- be set (since we're not in SMM mode):
2743	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
2744	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
2745	*/
2746
2747	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
2748	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
2749
2750	if ((fVal & fZap) == fVal)
2751	{ /* likely */ }
2752	else
2753	{
2754	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2755	g_HmMsrs.u.vmx.EntryCtls.n.allowed0, fVal, fZap));
2756	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_CTRL_ENTRY;
2757	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2758	}
2759
2760	/* Commit it to the VMCS. */
2761	if (pVmcsInfo->u32EntryCtls != fVal)
2762	{
2763	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY, fVal);
2764	AssertRC(rc);
2765	pVmcsInfo->u32EntryCtls = fVal;
2766	}
2767	}
2768
2769	/*
2770	* VM-exit controls.
2771	*/
2772	{
2773	uint32_t fVal = g_HmMsrs.u.vmx.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
2774	uint32_t const fZap = g_HmMsrs.u.vmx.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2775
2776	/*
2777	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
2778	* supported the 1-setting of this bit.
2779	*
2780	* For nested-guests, we set the "save debug controls" as the converse
2781	* "load debug controls" is mandatory for nested-guests anyway.
2782	*/
2783	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
2784
2785	/*
2786	* Set the host long mode active (EFER.LMA) bit (which Intel calls
2787	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
2788	* host EFER.LMA and EFER.LME bit to this value. See assertion in
2789	* vmxHCExportHostMsrs().
2790	*
2791	* For nested-guests, we always set this bit as we do not support 32-bit
2792	* hosts.
2793	*/
2794	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
2795
2796	#ifdef IN_RING0
2797	/*
2798	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
2799	*
2800	* For nested-guests, we should use the "save IA32_EFER" control if we also
2801	* used the "load IA32_EFER" control while exporting VM-entry controls.
2802	*/
2803	if ( g_fHmVmxSupportsVmcsEfer
2804	&& vmxHCShouldSwapEferMsr(pVCpu, pVmxTransient))
2805	{
2806	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
2807	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
2808	}
2809	#endif
2810
2811	/*
2812	* Enable saving of the VMX-preemption timer value on VM-exit.
2813	* For nested-guests, currently not exposed/used.
2814	*/
2815	/** @todo r=bird: Measure performance hit because of this vs. always rewriting
2816	* the timer value. */
2817	if (VM_IS_VMX_PREEMPT_TIMER_USED(pVM))
2818	{
2819	Assert(g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER);
2820	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
2821	}
2822
2823	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2824	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
2825
2826	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
2827	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
2828	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
2829
2830	if ((fVal & fZap) == fVal)
2831	{ /* likely */ }
2832	else
2833	{
2834	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2835	g_HmMsrs.u.vmx.ExitCtls.n.allowed0, fVal, fZap));
2836	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_CTRL_EXIT;
2837	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2838	}
2839
2840	/* Commit it to the VMCS. */
2841	if (pVmcsInfo->u32ExitCtls != fVal)
2842	{
2843	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXIT, fVal);
2844	AssertRC(rc);
2845	pVmcsInfo->u32ExitCtls = fVal;
2846	}
2847	}
2848
2849	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
2850	}
2851	return VINF_SUCCESS;
2852	}
2853
2854
2855	/**
2856	* Sets the TPR threshold in the VMCS.
2857	*
2858	* @param pVCpu The cross context virtual CPU structure.
2859	* @param pVmcsInfo The VMCS info. object.
2860	* @param u32TprThreshold The TPR threshold (task-priority class only).
2861	*/
2862	DECLINLINE(void) vmxHCApicSetTprThreshold(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
2863	{
2864	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
2865	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
2866	RT_NOREF(pVmcsInfo);
2867	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2868	AssertRC(rc);
2869	}
2870
2871
2872	/**
2873	* Exports the guest APIC TPR state into the VMCS.
2874	*
2875	* @param pVCpu The cross context virtual CPU structure.
2876	* @param pVmxTransient The VMX-transient structure.
2877	*
2878	* @remarks No-long-jump zone!!!
2879	*/
2880	static void vmxHCExportGuestApicTpr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
2881	{
2882	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
2883	{
2884	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
2885
2886	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2887	if (!pVmxTransient->fIsNestedGuest)
2888	{
2889	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
2890	&& APICIsEnabled(pVCpu))
2891	{
2892	/*
2893	* Setup TPR shadowing.
2894	*/
2895	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
2896	{
2897	bool fPendingIntr = false;
2898	uint8_t u8Tpr = 0;
2899	uint8_t u8PendingIntr = 0;
2900	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2901	AssertRC(rc);
2902
2903	/*
2904	* If there are interrupts pending but masked by the TPR, instruct VT-x to
2905	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
2906	* priority of the pending interrupt so we can deliver the interrupt. If there
2907	* are no interrupts pending, set threshold to 0 to not cause any
2908	* TPR-below-threshold VM-exits.
2909	*/
2910	uint32_t u32TprThreshold = 0;
2911	if (fPendingIntr)
2912	{
2913	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
2914	(which is the Task-Priority Class). */
2915	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
2916	const uint8_t u8TprPriority = u8Tpr >> 4;
2917	if (u8PendingPriority <= u8TprPriority)
2918	u32TprThreshold = u8PendingPriority;
2919	}
2920
2921	vmxHCApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
2922	}
2923	}
2924	}
2925	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
2926	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
2927	}
2928	}
2929
2930
2931	/**
2932	* Gets the guest interruptibility-state and updates related force-flags.
2933	*
2934	* @returns Guest's interruptibility-state.
2935	* @param pVCpu The cross context virtual CPU structure.
2936	*
2937	* @remarks No-long-jump zone!!!
2938	*/
2939	static uint32_t vmxHCGetGuestIntrStateAndUpdateFFs(PVMCPUCC pVCpu)
2940	{
2941	/*
2942	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
2943	*/
2944	uint32_t fIntrState = 0;
2945	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2946	{
2947	/* If inhibition is active, RIP and RFLAGS should've been imported from the VMCS already. */
2948	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
2949
2950	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2951	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
2952	{
2953	if (pCtx->eflags.Bits.u1IF)
2954	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
2955	else
2956	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
2957	}
2958	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2959	{
2960	/*
2961	* We can clear the inhibit force flag as even if we go back to the recompiler
2962	* without executing guest code in VT-x, the flag's condition to be cleared is
2963	* met and thus the cleared state is correct.
2964	*/
2965	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2966	}
2967	}
2968
2969	/*
2970	* Check if we should inhibit NMI delivery.
2971	*/
2972	if (CPUMIsGuestNmiBlocking(pVCpu))
2973	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
2974
2975	/*
2976	* Validate.
2977	*/
2978	#ifdef VBOX_STRICT
2979	/* We don't support block-by-SMI yet.*/
2980	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI));
2981
2982	/* Block-by-STI must not be set when interrupts are disabled. */
2983	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
2984	{
2985	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
2986	Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_IF);
2987	}
2988	#endif
2989
2990	return fIntrState;
2991	}
2992
2993
2994	/**
2995	* Exports the exception intercepts required for guest execution in the VMCS.
2996	*
2997	* @param pVCpu The cross context virtual CPU structure.
2998	* @param pVmxTransient The VMX-transient structure.
2999	*
3000	* @remarks No-long-jump zone!!!
3001	*/
3002	static void vmxHCExportGuestXcptIntercepts(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
3003	{
3004	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
3005	{
3006	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
3007	if ( !pVmxTransient->fIsNestedGuest
3008	&& VCPU_2_VMXSTATE(pVCpu).fGIMTrapXcptUD)
3009	vmxHCAddXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
3010	else
3011	vmxHCRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
3012
3013	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
3014	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
3015	}
3016	}
3017
3018
3019	/**
3020	* Exports the guest's RIP into the guest-state area in the VMCS.
3021	*
3022	* @param pVCpu The cross context virtual CPU structure.
3023	*
3024	* @remarks No-long-jump zone!!!
3025	*/
3026	static void vmxHCExportGuestRip(PVMCPUCC pVCpu)
3027	{
3028	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_RIP)
3029	{
3030	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
3031
3032	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
3033	AssertRC(rc);
3034
3035	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3036	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
3037	}
3038	}
3039
3040
3041	/**
3042	* Exports the guest's RSP into the guest-state area in the VMCS.
3043	*
3044	* @param pVCpu The cross context virtual CPU structure.
3045	*
3046	* @remarks No-long-jump zone!!!
3047	*/
3048	static void vmxHCExportGuestRsp(PVMCPUCC pVCpu)
3049	{
3050	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_RSP)
3051	{
3052	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
3053
3054	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
3055	AssertRC(rc);
3056
3057	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3058	Log4Func(("rsp=%#RX64\n", pVCpu->cpum.GstCtx.rsp));
3059	}
3060	}
3061
3062
3063	/**
3064	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
3065	*
3066	* @param pVCpu The cross context virtual CPU structure.
3067	* @param pVmxTransient The VMX-transient structure.
3068	*
3069	* @remarks No-long-jump zone!!!
3070	*/
3071	static void vmxHCExportGuestRflags(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
3072	{
3073	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3074	{
3075	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
3076
3077	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3078	Let us assert it as such and use 32-bit VMWRITE. */
3079	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
3080	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
3081	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3082	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
3083
3084	#ifdef IN_RING0
3085	/*
3086	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3087	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3088	* can run the real-mode guest code under Virtual 8086 mode.
3089	*/
3090	PVMXVMCSINFOSHARED pVmcsInfo = pVmxTransient->pVmcsInfo->pShared;
3091	if (pVmcsInfo->RealMode.fRealOnV86Active)
3092	{
3093	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3094	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3095	Assert(!pVmxTransient->fIsNestedGuest);
3096	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3097	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3098	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3099	}
3100	#else
3101	RT_NOREF(pVmxTransient);
3102	#endif
3103
3104	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3105	AssertRC(rc);
3106
3107	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3108	Log4Func(("eflags=%#RX32\n", fEFlags.u32));
3109	}
3110	}
3111
3112
3113	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3114	/**
3115	* Copies the nested-guest VMCS to the shadow VMCS.
3116	*
3117	* @returns VBox status code.
3118	* @param pVCpu The cross context virtual CPU structure.
3119	* @param pVmcsInfo The VMCS info. object.
3120	*
3121	* @remarks No-long-jump zone!!!
3122	*/
3123	static int vmxHCCopyNstGstToShadowVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3124	{
3125	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
3126	PCVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
3127
3128	/*
3129	* Disable interrupts so we don't get preempted while the shadow VMCS is the
3130	* current VMCS, as we may try saving guest lazy MSRs.
3131	*
3132	* Strictly speaking the lazy MSRs are not in the VMCS, but I'd rather not risk
3133	* calling the import VMCS code which is currently performing the guest MSR reads
3134	* (on 64-bit hosts) and accessing the auto-load/store MSR area on 32-bit hosts
3135	* and the rest of the VMX leave session machinery.
3136	*/
3137	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
3138
3139	int rc = vmxHCLoadShadowVmcs(pVmcsInfo);
3140	if (RT_SUCCESS(rc))
3141	{
3142	/*
3143	* Copy all guest read/write VMCS fields.
3144	*
3145	* We don't check for VMWRITE failures here for performance reasons and
3146	* because they are not expected to fail, barring irrecoverable conditions
3147	* like hardware errors.
3148	*/
3149	uint32_t const cShadowVmcsFields = pVM->hmr0.s.vmx.cShadowVmcsFields;
3150	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
3151	{
3152	uint64_t u64Val;
3153	uint32_t const uVmcsField = pVM->hmr0.s.vmx.paShadowVmcsFields[i];
3154	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
3155	VMX_VMCS_WRITE_64(pVCpu, uVmcsField, u64Val);
3156	}
3157
3158	/*
3159	* If the host CPU supports writing all VMCS fields, copy the guest read-only
3160	* VMCS fields, so the guest can VMREAD them without causing a VM-exit.
3161	*/
3162	if (g_HmMsrs.u.vmx.u64Misc & VMX_MISC_VMWRITE_ALL)
3163	{
3164	uint32_t const cShadowVmcsRoFields = pVM->hmr0.s.vmx.cShadowVmcsRoFields;
3165	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
3166	{
3167	uint64_t u64Val;
3168	uint32_t const uVmcsField = pVM->hmr0.s.vmx.paShadowVmcsRoFields[i];
3169	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
3170	VMX_VMCS_WRITE_64(pVCpu, uVmcsField, u64Val);
3171	}
3172	}
3173
3174	rc = vmxHCClearShadowVmcs(pVmcsInfo);
3175	rc \|= vmxHCLoadVmcs(pVmcsInfo);
3176	}
3177
3178	ASMSetFlags(fEFlags);
3179	return rc;
3180	}
3181
3182
3183	/**
3184	* Copies the shadow VMCS to the nested-guest VMCS.
3185	*
3186	* @returns VBox status code.
3187	* @param pVCpu The cross context virtual CPU structure.
3188	* @param pVmcsInfo The VMCS info. object.
3189	*
3190	* @remarks Called with interrupts disabled.
3191	*/
3192	static int vmxHCCopyShadowToNstGstVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3193	{
3194	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3195	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
3196	PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
3197
3198	int rc = vmxHCLoadShadowVmcs(pVmcsInfo);
3199	if (RT_SUCCESS(rc))
3200	{
3201	/*
3202	* Copy guest read/write fields from the shadow VMCS.
3203	* Guest read-only fields cannot be modified, so no need to copy them.
3204	*
3205	* We don't check for VMREAD failures here for performance reasons and
3206	* because they are not expected to fail, barring irrecoverable conditions
3207	* like hardware errors.
3208	*/
3209	uint32_t const cShadowVmcsFields = pVM->hmr0.s.vmx.cShadowVmcsFields;
3210	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
3211	{
3212	uint64_t u64Val;
3213	uint32_t const uVmcsField = pVM->hmr0.s.vmx.paShadowVmcsFields[i];
3214	VMX_VMCS_READ_64(pVCpu, uVmcsField, &u64Val);
3215	IEMWriteVmxVmcsField(pVmcsNstGst, uVmcsField, u64Val);
3216	}
3217
3218	rc = vmxHCClearShadowVmcs(pVmcsInfo);
3219	rc \|= vmxHCLoadVmcs(pVmcsInfo);
3220	}
3221	return rc;
3222	}
3223
3224
3225	/**
3226	* Enables VMCS shadowing for the given VMCS info. object.
3227	*
3228	* @param pVmcsInfo The VMCS info. object.
3229	*
3230	* @remarks No-long-jump zone!!!
3231	*/
3232	static void vmxHCEnableVmcsShadowing(PVMXVMCSINFO pVmcsInfo)
3233	{
3234	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
3235	if (!(uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING))
3236	{
3237	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
3238	uProcCtls2 \|= VMX_PROC_CTLS2_VMCS_SHADOWING;
3239	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc);
3240	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, pVmcsInfo->HCPhysShadowVmcs); AssertRC(rc);
3241	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
3242	pVmcsInfo->u64VmcsLinkPtr = pVmcsInfo->HCPhysShadowVmcs;
3243	Log4Func(("Enabled\n"));
3244	}
3245	}
3246
3247
3248	/**
3249	* Disables VMCS shadowing for the given VMCS info. object.
3250	*
3251	* @param pVmcsInfo The VMCS info. object.
3252	*
3253	* @remarks No-long-jump zone!!!
3254	*/
3255	static void vmxHCDisableVmcsShadowing(PVMXVMCSINFO pVmcsInfo)
3256	{
3257	/*
3258	* We want all VMREAD and VMWRITE instructions to cause VM-exits, so we clear the
3259	* VMCS shadowing control. However, VM-entry requires the shadow VMCS indicator bit
3260	* to match the VMCS shadowing control if the VMCS link pointer is not NIL_RTHCPHYS.
3261	* Hence, we must also reset the VMCS link pointer to ensure VM-entry does not fail.
3262	*
3263	* See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
3264	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
3265	*/
3266	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
3267	if (uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
3268	{
3269	uProcCtls2 &= ~VMX_PROC_CTLS2_VMCS_SHADOWING;
3270	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc);
3271	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS); AssertRC(rc);
3272	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
3273	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
3274	Log4Func(("Disabled\n"));
3275	}
3276	}
3277	#endif
3278
3279
3280	/**
3281	* Exports the guest hardware-virtualization state.
3282	*
3283	* @returns VBox status code.
3284	* @param pVCpu The cross context virtual CPU structure.
3285	* @param pVmxTransient The VMX-transient structure.
3286	*
3287	* @remarks No-long-jump zone!!!
3288	*/
3289	static int vmxHCExportGuestHwvirtState(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
3290	{
3291	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_HWVIRT)
3292	{
3293	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3294	/*
3295	* Check if the VMX feature is exposed to the guest and if the host CPU supports
3296	* VMCS shadowing.
3297	*/
3298	if (pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fUseVmcsShadowing)
3299	{
3300	/*
3301	* If the nested hypervisor has loaded a current VMCS and is in VMX root mode,
3302	* copy the nested hypervisor's current VMCS into the shadow VMCS and enable
3303	* VMCS shadowing to skip intercepting some or all VMREAD/VMWRITE VM-exits.
3304	*
3305	* We check for VMX root mode here in case the guest executes VMXOFF without
3306	* clearing the current VMCS pointer and our VMXOFF instruction emulation does
3307	* not clear the current VMCS pointer.
3308	*/
3309	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
3310	if ( CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx)
3311	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
3312	&& CPUMIsGuestVmxCurrentVmcsValid(&pVCpu->cpum.GstCtx))
3313	{
3314	/* Paranoia. */
3315	Assert(!pVmxTransient->fIsNestedGuest);
3316
3317	/*
3318	* For performance reasons, also check if the nested hypervisor's current VMCS
3319	* was newly loaded or modified before copying it to the shadow VMCS.
3320	*/
3321	if (!VCPU_2_VMXSTATE(pVCpu).vmx.fCopiedNstGstToShadowVmcs)
3322	{
3323	int rc = vmxHCCopyNstGstToShadowVmcs(pVCpu, pVmcsInfo);
3324	AssertRCReturn(rc, rc);
3325	VCPU_2_VMXSTATE(pVCpu).vmx.fCopiedNstGstToShadowVmcs = true;
3326	}
3327	vmxHCEnableVmcsShadowing(pVmcsInfo);
3328	}
3329	else
3330	vmxHCDisableVmcsShadowing(pVmcsInfo);
3331	}
3332	#else
3333	NOREF(pVmxTransient);
3334	#endif
3335	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_HWVIRT);
3336	}
3337	return VINF_SUCCESS;
3338	}
3339
3340
3341	/**
3342	* Exports the guest CR0 control register into the guest-state area in the VMCS.
3343	*
3344	* The guest FPU state is always pre-loaded hence we don't need to bother about
3345	* sharing FPU related CR0 bits between the guest and host.
3346	*
3347	* @returns VBox status code.
3348	* @param pVCpu The cross context virtual CPU structure.
3349	* @param pVmxTransient The VMX-transient structure.
3350	*
3351	* @remarks No-long-jump zone!!!
3352	*/
3353	static int vmxHCExportGuestCR0(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
3354	{
3355	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CR0)
3356	{
3357	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3358	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
3359
3360	uint64_t fSetCr0 = g_HmMsrs.u.vmx.u64Cr0Fixed0;
3361	uint64_t const fZapCr0 = g_HmMsrs.u.vmx.u64Cr0Fixed1;
3362	if (VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
3363	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
3364	else
3365	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3366
3367	if (!pVmxTransient->fIsNestedGuest)
3368	{
3369	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3370	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
3371	uint64_t const u64ShadowCr0 = u64GuestCr0;
3372	Assert(!RT_HI_U32(u64GuestCr0));
3373
3374	/*
3375	* Setup VT-x's view of the guest CR0.
3376	*/
3377	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
3378	if (VM_IS_VMX_NESTED_PAGING(pVM))
3379	{
3380	if (CPUMIsGuestPagingEnabled(pVCpu))
3381	{
3382	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3383	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
3384	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
3385	}
3386	else
3387	{
3388	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3389	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
3390	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3391	}
3392
3393	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3394	if (VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
3395	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
3396	}
3397	else
3398	{
3399	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3400	u64GuestCr0 \|= X86_CR0_WP;
3401	}
3402
3403	/*
3404	* Guest FPU bits.
3405	*
3406	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3407	* using CR0.TS.
3408	*
3409	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3410	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3411	*/
3412	u64GuestCr0 \|= X86_CR0_NE;
3413
3414	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3415	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
3416
3417	/*
3418	* Update exception intercepts.
3419	*/
3420	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
3421	#ifdef IN_RING0
3422	if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
3423	{
3424	Assert(PDMVmmDevHeapIsEnabled(pVM));
3425	Assert(pVM->hm.s.vmx.pRealModeTSS);
3426	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3427	}
3428	else
3429	#endif
3430	{
3431	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3432	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3433	if (fInterceptMF)
3434	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
3435	}
3436
3437	/* Additional intercepts for debugging, define these yourself explicitly. */
3438	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3439	uXcptBitmap \|= 0
3440	\| RT_BIT(X86_XCPT_BP)
3441	\| RT_BIT(X86_XCPT_DE)
3442	\| RT_BIT(X86_XCPT_NM)
3443	\| RT_BIT(X86_XCPT_TS)
3444	\| RT_BIT(X86_XCPT_UD)
3445	\| RT_BIT(X86_XCPT_NP)
3446	\| RT_BIT(X86_XCPT_SS)
3447	\| RT_BIT(X86_XCPT_GP)
3448	\| RT_BIT(X86_XCPT_PF)
3449	\| RT_BIT(X86_XCPT_MF)
3450	;
3451	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3452	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
3453	#endif
3454	if (VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv)
3455	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
3456	Assert(VM_IS_VMX_NESTED_PAGING(pVM) \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
3457
3458	/* Apply the hardware specified CR0 fixed bits and enable caching. */
3459	u64GuestCr0 \|= fSetCr0;
3460	u64GuestCr0 &= fZapCr0;
3461	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
3462
3463	/* Commit the CR0 and related fields to the guest VMCS. */
3464	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc);
3465	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc);
3466	if (uProcCtls != pVmcsInfo->u32ProcCtls)
3467	{
3468	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3469	AssertRC(rc);
3470	}
3471	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
3472	{
3473	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3474	AssertRC(rc);
3475	}
3476
3477	/* Update our caches. */
3478	pVmcsInfo->u32ProcCtls = uProcCtls;
3479	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3480
3481	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
3482	}
3483	else
3484	{
3485	/*
3486	* With nested-guests, we may have extended the guest/host mask here since we
3487	* merged in the outer guest's mask. Thus, the merged mask can include more bits
3488	* (to read from the nested-guest CR0 read-shadow) than the nested hypervisor
3489	* originally supplied. We must copy those bits from the nested-guest CR0 into
3490	* the nested-guest CR0 read-shadow.
3491	*/
3492	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3493	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
3494	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(&pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
3495	Assert(!RT_HI_U32(u64GuestCr0));
3496	Assert(u64GuestCr0 & X86_CR0_NE);
3497
3498	/* Apply the hardware specified CR0 fixed bits and enable caching. */
3499	u64GuestCr0 \|= fSetCr0;
3500	u64GuestCr0 &= fZapCr0;
3501	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
3502
3503	/* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
3504	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc);
3505	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc);
3506
3507	Log4Func(("cr0=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
3508	}
3509
3510	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3511	}
3512
3513	return VINF_SUCCESS;
3514	}
3515
3516
3517	/**
3518	* Exports the guest control registers (CR3, CR4) into the guest-state area
3519	* in the VMCS.
3520	*
3521	* @returns VBox strict status code.
3522	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3523	* without unrestricted guest access and the VMMDev is not presently
3524	* mapped (e.g. EFI32).
3525	*
3526	* @param pVCpu The cross context virtual CPU structure.
3527	* @param pVmxTransient The VMX-transient structure.
3528	*
3529	* @remarks No-long-jump zone!!!
3530	*/
3531	static VBOXSTRICTRC vmxHCExportGuestCR3AndCR4(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
3532	{
3533	int rc = VINF_SUCCESS;
3534	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3535
3536	/*
3537	* Guest CR2.
3538	* It's always loaded in the assembler code. Nothing to do here.
3539	*/
3540
3541	/*
3542	* Guest CR3.
3543	*/
3544	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CR3)
3545	{
3546	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
3547
3548	if (VM_IS_VMX_NESTED_PAGING(pVM))
3549	{
3550	#ifdef IN_RING0
3551	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
3552	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3553
3554	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3555	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
3556	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3557	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
3558
3559	/* VMX_EPT_MEMTYPE_WB support is already checked in vmxHCSetupTaggedTlb(). */
3560	pVmcsInfo->HCPhysEPTP \|= RT_BF_MAKE(VMX_BF_EPTP_MEMTYPE, VMX_EPTP_MEMTYPE_WB)
3561	\| RT_BF_MAKE(VMX_BF_EPTP_PAGE_WALK_LENGTH, VMX_EPTP_PAGE_WALK_LENGTH_4);
3562
3563	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3564	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3565	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3566	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
3567	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3568	\|\| (g_HmMsrs.u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_ACCESS_DIRTY),
3569	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
3570
3571	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
3572	AssertRC(rc);
3573	#endif
3574
3575	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3576	uint64_t u64GuestCr3 = pCtx->cr3;
3577	if ( VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
3578	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
3579	{
3580	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3581	if (CPUMIsGuestInPAEModeEx(pCtx))
3582	{
3583	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, pCtx->aPaePdpes[0].u); AssertRC(rc);
3584	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, pCtx->aPaePdpes[1].u); AssertRC(rc);
3585	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, pCtx->aPaePdpes[2].u); AssertRC(rc);
3586	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, pCtx->aPaePdpes[3].u); AssertRC(rc);
3587	}
3588
3589	/*
3590	* The guest's view of its CR3 is unblemished with nested paging when the
3591	* guest is using paging or we have unrestricted guest execution to handle
3592	* the guest when it's not using paging.
3593	*/
3594	}
3595	#ifdef IN_RING0
3596	else
3597	{
3598	/*
3599	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
3600	* thinks it accesses physical memory directly, we use our identity-mapped
3601	* page table to map guest-linear to guest-physical addresses. EPT takes care
3602	* of translating it to host-physical addresses.
3603	*/
3604	RTGCPHYS GCPhys;
3605	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3606
3607	/* We obtain it here every time as the guest could have relocated this PCI region. */
3608	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3609	if (RT_SUCCESS(rc))
3610	{ /* likely */ }
3611	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3612	{
3613	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3614	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3615	}
3616	else
3617	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3618
3619	u64GuestCr3 = GCPhys;
3620	}
3621	#endif
3622
3623	Log4Func(("guest_cr3=%#RX64 (GstN)\n", u64GuestCr3));
3624	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR3, u64GuestCr3);
3625	AssertRC(rc);
3626	}
3627	else
3628	{
3629	Assert(!pVmxTransient->fIsNestedGuest);
3630	/* Non-nested paging case, just use the hypervisor's CR3. */
3631	RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);
3632
3633	Log4Func(("guest_cr3=%#RX64 (HstN)\n", HCPhysGuestCr3));
3634	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
3635	AssertRC(rc);
3636	}
3637
3638	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3639	}
3640
3641	/*
3642	* Guest CR4.
3643	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
3644	*/
3645	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CR4)
3646	{
3647	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3648	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
3649
3650	uint64_t const fSetCr4 = g_HmMsrs.u.vmx.u64Cr4Fixed0;
3651	uint64_t const fZapCr4 = g_HmMsrs.u.vmx.u64Cr4Fixed1;
3652
3653	/*
3654	* With nested-guests, we may have extended the guest/host mask here (since we
3655	* merged in the outer guest's mask, see vmxHCMergeVmcsNested). This means, the
3656	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
3657	* the nested hypervisor originally supplied. Thus, we should, in essence, copy
3658	* those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
3659	*/
3660	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
3661	uint64_t u64GuestCr4 = pCtx->cr4;
3662	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
3663	? pCtx->cr4
3664	: CPUMGetGuestVmxMaskedCr4(pCtx, pVmcsInfo->u64Cr4Mask);
3665	Assert(!RT_HI_U32(u64GuestCr4));
3666
3667	#ifdef IN_RING0
3668	/*
3669	* Setup VT-x's view of the guest CR4.
3670	*
3671	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3672	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3673	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3674	*
3675	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3676	*/
3677	if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
3678	{
3679	Assert(pVM->hm.s.vmx.pRealModeTSS);
3680	Assert(PDMVmmDevHeapIsEnabled(pVM));
3681	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
3682	}
3683	#endif
3684
3685	if (VM_IS_VMX_NESTED_PAGING(pVM))
3686	{
3687	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3688	&& !VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
3689	{
3690	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3691	u64GuestCr4 \|= X86_CR4_PSE;
3692	/* Our identity mapping is a 32-bit page directory. */
3693	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
3694	}
3695	/* else use guest CR4.*/
3696	}
3697	else
3698	{
3699	Assert(!pVmxTransient->fIsNestedGuest);
3700
3701	/*
3702	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3703	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3704	*/
3705	switch (VCPU_2_VMXSTATE(pVCpu).enmShadowMode)
3706	{
3707	case PGMMODE_REAL: /* Real-mode. */
3708	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3709	case PGMMODE_32_BIT: /* 32-bit paging. */
3710	{
3711	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
3712	break;
3713	}
3714
3715	case PGMMODE_PAE: /* PAE paging. */
3716	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3717	{
3718	u64GuestCr4 \|= X86_CR4_PAE;
3719	break;
3720	}
3721
3722	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3723	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3724	{
3725	#ifdef VBOX_WITH_64_BITS_GUESTS
3726	/* For our assumption in vmxHCShouldSwapEferMsr. */
3727	Assert(u64GuestCr4 & X86_CR4_PAE);
3728	break;
3729	#endif
3730	}
3731	default:
3732	AssertFailed();
3733	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3734	}
3735	}
3736
3737	/* Apply the hardware specified CR4 fixed bits (mainly CR4.VMXE). */
3738	u64GuestCr4 \|= fSetCr4;
3739	u64GuestCr4 &= fZapCr4;
3740
3741	/* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
3742	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR4, u64GuestCr4); AssertRC(rc);
3743	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4); AssertRC(rc);
3744
3745	#ifdef IN_RING0
3746	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
3747	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
3748	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
3749	{
3750	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
3751	vmxHCUpdateStartVmFunction(pVCpu);
3752	}
3753	#endif
3754
3755	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CR4);
3756
3757	Log4Func(("cr4=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
3758	}
3759	return rc;
3760	}
3761
3762
3763	/**
3764	* Exports the guest debug registers into the guest-state area in the VMCS.
3765	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3766	*
3767	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
3768	*
3769	* @returns VBox status code.
3770	* @param pVCpu The cross context virtual CPU structure.
3771	* @param pVmxTransient The VMX-transient structure.
3772	*
3773	* @remarks No-long-jump zone!!!
3774	*/
3775	static int vmxHCExportSharedDebugState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
3776	{
3777	#ifdef IN_RING0
3778	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3779	#endif
3780
3781	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
3782	* stepping. */
3783	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
3784	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3785	if (pVmxTransient->fIsNestedGuest)
3786	{
3787	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
3788	AssertRC(rc);
3789
3790	/*
3791	* We don't want to always intercept MOV DRx for nested-guests as it causes
3792	* problems when the nested hypervisor isn't intercepting them, see @bugref{10080}.
3793	* Instead, they are strictly only requested when the nested hypervisor intercepts
3794	* them -- handled while merging VMCS controls.
3795	*
3796	* If neither the outer nor the nested-hypervisor is intercepting MOV DRx,
3797	* then the nested-guest debug state should be actively loaded on the host so that
3798	* nested-guest reads its own debug registers without causing VM-exits.
3799	*/
3800	if ( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT)
3801	&& !CPUMIsGuestDebugStateActive(pVCpu))
3802	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3803	return VINF_SUCCESS;
3804	}
3805	#endif
3806
3807	#ifdef VBOX_STRICT
3808	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3809	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
3810	{
3811	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3812	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
3813	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
3814	}
3815	#endif
3816
3817	#ifdef IN_RING0 /** @todo */
3818	bool fSteppingDB = false;
3819	bool fInterceptMovDRx = false;
3820	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
3821	if (VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
3822	{
3823	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3824	if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
3825	{
3826	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
3827	Assert(fSteppingDB == false);
3828	}
3829	else
3830	{
3831	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
3832	VCPU_2_VMXSTATE(pVCpu).fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
3833	pVCpu->hmr0.s.fClearTrapFlag = true;
3834	fSteppingDB = true;
3835	}
3836	}
3837
3838	uint64_t u64GuestDr7;
3839	if ( fSteppingDB
3840	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3841	{
3842	/*
3843	* Use the combined guest and host DRx values found in the hypervisor register set
3844	* because the hypervisor debugger has breakpoints active or someone is single stepping
3845	* on the host side without a monitor trap flag.
3846	*
3847	* Note! DBGF expects a clean DR6 state before executing guest code.
3848	*/
3849	if (!CPUMIsHyperDebugStateActive(pVCpu))
3850	{
3851	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3852	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3853	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3854	}
3855
3856	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
3857	u64GuestDr7 = CPUMGetHyperDR7(pVCpu);
3858	pVCpu->hmr0.s.fUsingHyperDR7 = true;
3859	fInterceptMovDRx = true;
3860	}
3861	else
3862	{
3863	/*
3864	* If the guest has enabled debug registers, we need to load them prior to
3865	* executing guest code so they'll trigger at the right time.
3866	*/
3867	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
3868	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3869	{
3870	if (!CPUMIsGuestDebugStateActive(pVCpu))
3871	{
3872	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3873	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3874	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3875	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatDRxArmed);
3876	}
3877	Assert(!fInterceptMovDRx);
3878	}
3879	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3880	{
3881	/*
3882	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3883	* must intercept #DB in order to maintain a correct DR6 guest value, and
3884	* because we need to intercept it to prevent nested #DBs from hanging the
3885	* CPU, we end up always having to intercept it. See vmxHCSetupVmcsXcptBitmap().
3886	*/
3887	fInterceptMovDRx = true;
3888	}
3889
3890	/* Update DR7 with the actual guest value. */
3891	u64GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
3892	pVCpu->hmr0.s.fUsingHyperDR7 = false;
3893	}
3894
3895	if (fInterceptMovDRx)
3896	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
3897	else
3898	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
3899
3900	/*
3901	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
3902	* monitor-trap flag and update our cache.
3903	*/
3904	if (uProcCtls != pVmcsInfo->u32ProcCtls)
3905	{
3906	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3907	AssertRC(rc);
3908	pVmcsInfo->u32ProcCtls = uProcCtls;
3909	}
3910
3911	/*
3912	* Update guest DR7.
3913	*/
3914	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_DR7, u64GuestDr7);
3915	AssertRC(rc);
3916
3917	/*
3918	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
3919	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
3920	*
3921	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
3922	*/
3923	if (fSteppingDB)
3924	{
3925	Assert(VCPU_2_VMXSTATE(pVCpu).fSingleInstruction);
3926	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
3927
3928	uint32_t fIntrState = 0;
3929	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
3930	AssertRC(rc);
3931
3932	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
3933	{
3934	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
3935	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, fIntrState);
3936	AssertRC(rc);
3937	}
3938	}
3939	#endif /* !IN_RING0 */
3940
3941	return VINF_SUCCESS;
3942	}
3943
3944
3945	#ifdef VBOX_STRICT
3946	/**
3947	* Strict function to validate segment registers.
3948	*
3949	* @param pVCpu The cross context virtual CPU structure.
3950	* @param pVmcsInfo The VMCS info. object.
3951	*
3952	* @remarks Will import guest CR0 on strict builds during validation of
3953	* segments.
3954	*/
3955	static void vmxHCValidateSegmentRegs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3956	{
3957	/*
3958	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
3959	*
3960	* The reason we check for attribute value 0 in this function and not just the unusable bit is
3961	* because vmxHCExportGuestSegReg() only updates the VMCS' copy of the value with the
3962	* unusable bit and doesn't change the guest-context value.
3963	*/
3964	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3965	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3966	vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
3967	if ( !VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
3968	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3969	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3970	{
3971	/* Protected mode checks */
3972	/* CS */
3973	Assert(pCtx->cs.Attr.n.u1Present);
3974	Assert(!(pCtx->cs.Attr.u & 0xf00));
3975	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3976	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3977	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3978	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3979	\|\| (pCtx->cs.Attr.n.u1Granularity));
3980	/* CS cannot be loaded with NULL in protected mode. */
3981	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
3982	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3983	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3984	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3985	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3986	else
3987	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3988	/* SS */
3989	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3990	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3991	if ( !(pCtx->cr0 & X86_CR0_PE)
3992	\|\| pCtx->cs.Attr.n.u4Type == 3)
3993	{
3994	Assert(!pCtx->ss.Attr.n.u2Dpl);
3995	}
3996	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3997	{
3998	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3999	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4000	Assert(pCtx->ss.Attr.n.u1Present);
4001	Assert(!(pCtx->ss.Attr.u & 0xf00));
4002	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4003	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4004	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4005	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4006	\|\| (pCtx->ss.Attr.n.u1Granularity));
4007	}
4008	/* DS, ES, FS, GS - only check for usable selectors, see vmxHCExportGuestSegReg(). */
4009	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4010	{
4011	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4012	Assert(pCtx->ds.Attr.n.u1Present);
4013	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4014	Assert(!(pCtx->ds.Attr.u & 0xf00));
4015	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4016	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4017	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4018	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4019	\|\| (pCtx->ds.Attr.n.u1Granularity));
4020	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4021	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4022	}
4023	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4024	{
4025	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4026	Assert(pCtx->es.Attr.n.u1Present);
4027	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4028	Assert(!(pCtx->es.Attr.u & 0xf00));
4029	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4030	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4031	\|\| !(pCtx->es.Attr.n.u1Granularity));
4032	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4033	\|\| (pCtx->es.Attr.n.u1Granularity));
4034	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4035	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4036	}
4037	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4038	{
4039	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4040	Assert(pCtx->fs.Attr.n.u1Present);
4041	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4042	Assert(!(pCtx->fs.Attr.u & 0xf00));
4043	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4044	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4045	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4046	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4047	\|\| (pCtx->fs.Attr.n.u1Granularity));
4048	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4049	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4050	}
4051	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4052	{
4053	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4054	Assert(pCtx->gs.Attr.n.u1Present);
4055	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4056	Assert(!(pCtx->gs.Attr.u & 0xf00));
4057	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4058	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4059	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4060	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4061	\|\| (pCtx->gs.Attr.n.u1Granularity));
4062	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4063	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4064	}
4065	/* 64-bit capable CPUs. */
4066	Assert(!RT_HI_U32(pCtx->cs.u64Base));
4067	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
4068	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
4069	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
4070	}
4071	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4072	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4073	&& !VM_IS_VMX_UNRESTRICTED_GUEST(pVM)))
4074	{
4075	/* Real and v86 mode checks. */
4076	/* vmxHCExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4077	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4078	#ifdef IN_RING0
4079	if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
4080	{
4081	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
4082	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4083	}
4084	else
4085	#endif
4086	{
4087	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4088	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4089	}
4090
4091	/* CS */
4092	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4093	Assert(pCtx->cs.u32Limit == 0xffff);
4094	Assert(u32CSAttr == 0xf3);
4095	/* SS */
4096	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4097	Assert(pCtx->ss.u32Limit == 0xffff);
4098	Assert(u32SSAttr == 0xf3);
4099	/* DS */
4100	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4101	Assert(pCtx->ds.u32Limit == 0xffff);
4102	Assert(u32DSAttr == 0xf3);
4103	/* ES */
4104	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4105	Assert(pCtx->es.u32Limit == 0xffff);
4106	Assert(u32ESAttr == 0xf3);
4107	/* FS */
4108	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4109	Assert(pCtx->fs.u32Limit == 0xffff);
4110	Assert(u32FSAttr == 0xf3);
4111	/* GS */
4112	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4113	Assert(pCtx->gs.u32Limit == 0xffff);
4114	Assert(u32GSAttr == 0xf3);
4115	/* 64-bit capable CPUs. */
4116	Assert(!RT_HI_U32(pCtx->cs.u64Base));
4117	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
4118	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
4119	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
4120	}
4121	}
4122	#endif /* VBOX_STRICT */
4123
4124
4125	/**
4126	* Exports a guest segment register into the guest-state area in the VMCS.
4127	*
4128	* @returns VBox status code.
4129	* @param pVCpu The cross context virtual CPU structure.
4130	* @param pVmcsInfo The VMCS info. object.
4131	* @param iSegReg The segment register number (X86_SREG_XXX).
4132	* @param pSelReg Pointer to the segment selector.
4133	*
4134	* @remarks No-long-jump zone!!!
4135	*/
4136	static int vmxHCExportGuestSegReg(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, uint32_t iSegReg, PCCPUMSELREG pSelReg)
4137	{
4138	Assert(iSegReg < X86_SREG_COUNT);
4139
4140	uint32_t u32Access = pSelReg->Attr.u;
4141	#ifdef IN_RING0
4142	if (!pVmcsInfo->pShared->RealMode.fRealOnV86Active)
4143	#endif
4144	{
4145	/*
4146	* The way to differentiate between whether this is really a null selector or was just
4147	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
4148	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4149	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4150	* NULL selectors loaded in protected-mode have their attribute as 0.
4151	*/
4152	if (u32Access)
4153	{ }
4154	else
4155	u32Access = X86DESCATTR_UNUSABLE;
4156	}
4157	#ifdef IN_RING0
4158	else
4159	{
4160	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4161	u32Access = 0xf3;
4162	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4163	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4164	RT_NOREF_PV(pVCpu);
4165	}
4166	#else
4167	RT_NOREF(pVmcsInfo);
4168	#endif
4169
4170	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4171	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4172	("Access bit not set for usable segment. %.2s sel=%#x attr %#x\n", "ESCSSSDSFSGS" + iSegReg * 2, pSelReg, pSelReg->Attr.u));
4173
4174	/*
4175	* Commit it to the VMCS.
4176	*/
4177	Assert((uint32_t)VMX_VMCS16_GUEST_SEG_SEL(iSegReg) == g_aVmcsSegSel[iSegReg]);
4178	Assert((uint32_t)VMX_VMCS32_GUEST_SEG_LIMIT(iSegReg) == g_aVmcsSegLimit[iSegReg]);
4179	Assert((uint32_t)VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(iSegReg) == g_aVmcsSegAttr[iSegReg]);
4180	Assert((uint32_t)VMX_VMCS_GUEST_SEG_BASE(iSegReg) == g_aVmcsSegBase[iSegReg]);
4181	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS16_GUEST_SEG_SEL(iSegReg), pSelReg->Sel); AssertRC(rc);
4182	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_SEG_LIMIT(iSegReg), pSelReg->u32Limit); AssertRC(rc);
4183	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_SEG_BASE(iSegReg), pSelReg->u64Base); AssertRC(rc);
4184	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(iSegReg), u32Access); AssertRC(rc);
4185	return VINF_SUCCESS;
4186	}
4187
4188
4189	/**
4190	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
4191	* area in the VMCS.
4192	*
4193	* @returns VBox status code.
4194	* @param pVCpu The cross context virtual CPU structure.
4195	* @param pVmxTransient The VMX-transient structure.
4196	*
4197	* @remarks Will import guest CR0 on strict builds during validation of
4198	* segments.
4199	* @remarks No-long-jump zone!!!
4200	*/
4201	static int vmxHCExportGuestSegRegsXdtr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
4202	{
4203	int rc = VERR_INTERNAL_ERROR_5;
4204	#ifdef IN_RING0
4205	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4206	#endif
4207	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4208	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4209	#ifdef IN_RING0
4210	PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
4211	#endif
4212
4213	/*
4214	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4215	*/
4216	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4217	{
4218	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CS)
4219	{
4220	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
4221	#ifdef IN_RING0
4222	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
4223	pVmcsInfoShared->RealMode.AttrCS.u = pCtx->cs.Attr.u;
4224	#endif
4225	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
4226	AssertRC(rc);
4227	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CS);
4228	}
4229
4230	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_SS)
4231	{
4232	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
4233	#ifdef IN_RING0
4234	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
4235	pVmcsInfoShared->RealMode.AttrSS.u = pCtx->ss.Attr.u;
4236	#endif
4237	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
4238	AssertRC(rc);
4239	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_SS);
4240	}
4241
4242	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_DS)
4243	{
4244	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
4245	#ifdef IN_RING0
4246	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
4247	pVmcsInfoShared->RealMode.AttrDS.u = pCtx->ds.Attr.u;
4248	#endif
4249	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
4250	AssertRC(rc);
4251	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_DS);
4252	}
4253
4254	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_ES)
4255	{
4256	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
4257	#ifdef IN_RING0
4258	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
4259	pVmcsInfoShared->RealMode.AttrES.u = pCtx->es.Attr.u;
4260	#endif
4261	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
4262	AssertRC(rc);
4263	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_ES);
4264	}
4265
4266	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_FS)
4267	{
4268	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
4269	#ifdef IN_RING0
4270	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
4271	pVmcsInfoShared->RealMode.AttrFS.u = pCtx->fs.Attr.u;
4272	#endif
4273	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
4274	AssertRC(rc);
4275	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_FS);
4276	}
4277
4278	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_GS)
4279	{
4280	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
4281	#ifdef IN_RING0
4282	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
4283	pVmcsInfoShared->RealMode.AttrGS.u = pCtx->gs.Attr.u;
4284	#endif
4285	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
4286	AssertRC(rc);
4287	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_GS);
4288	}
4289
4290	#ifdef VBOX_STRICT
4291	vmxHCValidateSegmentRegs(pVCpu, pVmcsInfo);
4292	#endif
4293	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
4294	pCtx->cs.Attr.u));
4295	}
4296
4297	/*
4298	* Guest TR.
4299	*/
4300	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_TR)
4301	{
4302	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
4303
4304	/*
4305	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4306	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4307	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4308	*/
4309	uint16_t u16Sel;
4310	uint32_t u32Limit;
4311	uint64_t u64Base;
4312	uint32_t u32AccessRights;
4313	#ifdef IN_RING0
4314	if (!pVmcsInfoShared->RealMode.fRealOnV86Active)
4315	#endif
4316	{
4317	u16Sel = pCtx->tr.Sel;
4318	u32Limit = pCtx->tr.u32Limit;
4319	u64Base = pCtx->tr.u64Base;
4320	u32AccessRights = pCtx->tr.Attr.u;
4321	}
4322	#ifdef IN_RING0
4323	else
4324	{
4325	Assert(!pVmxTransient->fIsNestedGuest);
4326	Assert(pVM->hm.s.vmx.pRealModeTSS);
4327	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
4328
4329	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4330	RTGCPHYS GCPhys;
4331	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4332	AssertRCReturn(rc, rc);
4333
4334	X86DESCATTR DescAttr;
4335	DescAttr.u = 0;
4336	DescAttr.n.u1Present = 1;
4337	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4338
4339	u16Sel = 0;
4340	u32Limit = HM_VTX_TSS_SIZE;
4341	u64Base = GCPhys;
4342	u32AccessRights = DescAttr.u;
4343	}
4344	#endif
4345
4346	/* Validate. */
4347	Assert(!(u16Sel & RT_BIT(2)));
4348	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4349	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4350	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4351	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4352	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4353	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4354	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4355	Assert( (u32Limit & 0xfff) == 0xfff
4356	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4357	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
4358	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4359
4360	rc = VMX_VMCS_WRITE_16(pVCpu, VMX_VMCS16_GUEST_TR_SEL, u16Sel); AssertRC(rc);
4361	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRC(rc);
4362	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRC(rc);
4363	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRC(rc);
4364
4365	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_TR);
4366	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
4367	}
4368
4369	/*
4370	* Guest GDTR.
4371	*/
4372	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4373	{
4374	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
4375
4376	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt); AssertRC(rc);
4377	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt); AssertRC(rc);
4378
4379	/* Validate. */
4380	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4381
4382	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4383	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
4384	}
4385
4386	/*
4387	* Guest LDTR.
4388	*/
4389	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4390	{
4391	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
4392
4393	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4394	uint32_t u32Access;
4395	if ( !pVmxTransient->fIsNestedGuest
4396	&& !pCtx->ldtr.Attr.u)
4397	u32Access = X86DESCATTR_UNUSABLE;
4398	else
4399	u32Access = pCtx->ldtr.Attr.u;
4400
4401	rc = VMX_VMCS_WRITE_16(pVCpu, VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel); AssertRC(rc);
4402	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit); AssertRC(rc);
4403	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRC(rc);
4404	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base); AssertRC(rc);
4405
4406	/* Validate. */
4407	if (!(u32Access & X86DESCATTR_UNUSABLE))
4408	{
4409	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4410	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4411	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4412	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4413	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4414	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4415	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
4416	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4417	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
4418	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4419	}
4420
4421	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4422	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
4423	}
4424
4425	/*
4426	* Guest IDTR.
4427	*/
4428	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4429	{
4430	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
4431
4432	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt); AssertRC(rc);
4433	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt); AssertRC(rc);
4434
4435	/* Validate. */
4436	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4437
4438	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4439	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
4440	}
4441
4442	return VINF_SUCCESS;
4443	}
4444
4445
4446	/**
4447	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4448	* areas.
4449	*
4450	* These MSRs will automatically be loaded to the host CPU on every successful
4451	* VM-entry and stored from the host CPU on every successful VM-exit.
4452	*
4453	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
4454	* actual host MSR values are not- updated here for performance reasons. See
4455	* vmxHCExportHostMsrs().
4456	*
4457	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4458	*
4459	* @returns VBox status code.
4460	* @param pVCpu The cross context virtual CPU structure.
4461	* @param pVmxTransient The VMX-transient structure.
4462	*
4463	* @remarks No-long-jump zone!!!
4464	*/
4465	static int vmxHCExportGuestMsrs(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
4466	{
4467	AssertPtr(pVCpu);
4468	AssertPtr(pVmxTransient);
4469
4470	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4471	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4472
4473	/*
4474	* MSRs that we use the auto-load/store MSR area in the VMCS.
4475	* For 64-bit hosts, we load/restore them lazily, see vmxHCLazyLoadGuestMsrs(),
4476	* nothing to do here. The host MSR values are updated when it's safe in
4477	* vmxHCLazySaveHostMsrs().
4478	*
4479	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
4480	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
4481	* emulation. The merged MSR permission bitmap will ensure that we get VM-exits
4482	* for any MSR that are not part of the lazy MSRs so we do not need to place
4483	* those MSRs into the auto-load/store MSR area. Nothing to do here.
4484	*/
4485	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4486	{
4487	/* No auto-load/store MSRs currently. */
4488	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4489	}
4490
4491	/*
4492	* Guest Sysenter MSRs.
4493	*/
4494	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4495	{
4496	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4497
4498	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4499	{
4500	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
4501	AssertRC(rc);
4502	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4503	}
4504
4505	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4506	{
4507	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
4508	AssertRC(rc);
4509	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4510	}
4511
4512	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4513	{
4514	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
4515	AssertRC(rc);
4516	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4517	}
4518	}
4519
4520	/*
4521	* Guest/host EFER MSR.
4522	*/
4523	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4524	{
4525	/* Whether we are using the VMCS to swap the EFER MSR must have been
4526	determined earlier while exporting VM-entry/VM-exit controls. */
4527	Assert(!(ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
4528	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
4529
4530	if (vmxHCShouldSwapEferMsr(pVCpu, pVmxTransient))
4531	{
4532	/*
4533	* EFER.LME is written by software, while EFER.LMA is set by the CPU to (CR0.PG & EFER.LME).
4534	* This means a guest can set EFER.LME=1 while CR0.PG=0 and EFER.LMA can remain 0.
4535	* VT-x requires that "IA-32e mode guest" VM-entry control must be identical to EFER.LMA
4536	* and to CR0.PG. Without unrestricted execution, CR0.PG (used for VT-x, not the shadow)
4537	* must always be 1. This forces us to effectively clear both EFER.LMA and EFER.LME until
4538	* the guest has also set CR0.PG=1. Otherwise, we would run into an invalid-guest state
4539	* during VM-entry.
4540	*/
4541	uint64_t uGuestEferMsr = pCtx->msrEFER;
4542	if (!VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
4543	{
4544	if (!(pCtx->msrEFER & MSR_K6_EFER_LMA))
4545	uGuestEferMsr &= ~MSR_K6_EFER_LME;
4546	else
4547	Assert((pCtx->msrEFER & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
4548	}
4549
4550	/*
4551	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4552	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4553	*/
4554	if (g_fHmVmxSupportsVmcsEfer)
4555	{
4556	int rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_EFER_FULL, uGuestEferMsr);
4557	AssertRC(rc);
4558	}
4559	else
4560	{
4561	/*
4562	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
4563	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
4564	*/
4565	int rc = vmxHCAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, uGuestEferMsr,
4566	false /* fSetReadWrite /, false / fUpdateHostMsr */);
4567	AssertRCReturn(rc, rc);
4568	}
4569
4570	Log4Func(("efer=%#RX64 shadow=%#RX64\n", uGuestEferMsr, pCtx->msrEFER));
4571	}
4572	else if (!g_fHmVmxSupportsVmcsEfer)
4573	vmxHCRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
4574
4575	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4576	}
4577
4578	/*
4579	* Other MSRs.
4580	*/
4581	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
4582	{
4583	/* Speculation Control (R/W). */
4584	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
4585	if (pVM->cpum.ro.GuestFeatures.fIbrs)
4586	{
4587	int rc = vmxHCAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
4588	false /* fSetReadWrite /, false / fUpdateHostMsr */);
4589	AssertRCReturn(rc, rc);
4590	}
4591
4592	#ifdef IN_RING0 /** @todo */
4593	/* Last Branch Record. */
4594	if (VM_IS_VMX_LBR(pVM))
4595	{
4596	PVMXVMCSINFOSHARED const pVmcsInfoShared = pVmxTransient->pVmcsInfo->pShared;
4597	uint32_t const idFromIpMsrStart = pVM->hmr0.s.vmx.idLbrFromIpMsrFirst;
4598	uint32_t const idToIpMsrStart = pVM->hmr0.s.vmx.idLbrToIpMsrFirst;
4599	uint32_t const cLbrStack = pVM->hmr0.s.vmx.idLbrFromIpMsrLast - pVM->hmr0.s.vmx.idLbrFromIpMsrFirst + 1;
4600	Assert(cLbrStack <= 32);
4601	for (uint32_t i = 0; i < cLbrStack; i++)
4602	{
4603	int rc = vmxHCAddAutoLoadStoreMsr(pVCpu, pVmxTransient, idFromIpMsrStart + i,
4604	pVmcsInfoShared->au64LbrFromIpMsr[i],
4605	false /* fSetReadWrite /, false / fUpdateHostMsr */);
4606	AssertRCReturn(rc, rc);
4607
4608	/* Some CPUs don't have a Branch-To-IP MSR (P4 and related Xeons). */
4609	if (idToIpMsrStart != 0)
4610	{
4611	rc = vmxHCAddAutoLoadStoreMsr(pVCpu, pVmxTransient, idToIpMsrStart + i,
4612	pVmcsInfoShared->au64LbrToIpMsr[i],
4613	false /* fSetReadWrite /, false / fUpdateHostMsr */);
4614	AssertRCReturn(rc, rc);
4615	}
4616	}
4617
4618	/* Add LBR top-of-stack MSR (which contains the index to the most recent record). */
4619	int rc = vmxHCAddAutoLoadStoreMsr(pVCpu, pVmxTransient, pVM->hmr0.s.vmx.idLbrTosMsr,
4620	pVmcsInfoShared->u64LbrTosMsr, false /* fSetReadWrite */,
4621	false /* fUpdateHostMsr */);
4622	AssertRCReturn(rc, rc);
4623	}
4624	#endif /* !IN_RING0 */
4625
4626	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
4627	}
4628
4629	return VINF_SUCCESS;
4630	}
4631
4632
4633	#ifdef IN_RING0
4634	/**
4635	* Sets up the usage of TSC-offsetting and updates the VMCS.
4636	*
4637	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
4638	* VMX-preemption timer.
4639	*
4640	* @returns VBox status code.
4641	* @param pVCpu The cross context virtual CPU structure.
4642	* @param pVmxTransient The VMX-transient structure.
4643	* @param idCurrentCpu The current CPU number.
4644	*
4645	* @remarks No-long-jump zone!!!
4646	*/
4647	static void vmxHCUpdateTscOffsettingAndPreemptTimer(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, RTCPUID idCurrentCpu)
4648	{
4649	bool fOffsettedTsc;
4650	bool fParavirtTsc;
4651	uint64_t uTscOffset;
4652	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4653	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
4654
4655	if (VM_IS_VMX_PREEMPT_TIMER_USED(pVM))
4656	{
4657	/* The TMCpuTickGetDeadlineAndTscOffset function is expensive (calling it on
4658	every entry slowed down the bs2-test1 CPUID testcase by ~33% (on an 10980xe). */
4659	uint64_t cTicksToDeadline;
4660	if ( idCurrentCpu == pVCpu->hmr0.s.idLastCpu
4661	&& TMVirtualSyncIsCurrentDeadlineVersion(pVM, pVCpu->hmr0.s.vmx.uTscDeadlineVersion))
4662	{
4663	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatVmxPreemptionReusingDeadline);
4664	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
4665	cTicksToDeadline = pVCpu->hmr0.s.vmx.uTscDeadline - SUPReadTsc();
4666	if ((int64_t)cTicksToDeadline > 0)
4667	{ /* hopefully */ }
4668	else
4669	{
4670	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatVmxPreemptionReusingDeadlineExpired);
4671	cTicksToDeadline = 0;
4672	}
4673	}
4674	else
4675	{
4676	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatVmxPreemptionRecalcingDeadline);
4677	cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc,
4678	&pVCpu->hmr0.s.vmx.uTscDeadline,
4679	&pVCpu->hmr0.s.vmx.uTscDeadlineVersion);
4680	pVCpu->hmr0.s.vmx.uTscDeadline += cTicksToDeadline;
4681	if (cTicksToDeadline >= 128)
4682	{ /* hopefully */ }
4683	else
4684	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatVmxPreemptionRecalcingDeadlineExpired);
4685	}
4686
4687	/* Make sure the returned values have sane upper and lower boundaries. */
4688	uint64_t const u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
4689	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second, 15.625ms. / /* @todo r=bird: Once real+virtual timers move to separate thread, we can raise the upper limit (16ms isn't much). ASSUMES working poke cpu function. */
4690	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 32678); /* 1/32768th of a second, ~30us. */
4691	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4692
4693	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
4694	* preemption timers here. We probably need to clamp the preemption timer,
4695	* after converting the timer value to the host. */
4696	uint32_t const cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4697	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
4698	AssertRC(rc);
4699	}
4700	else
4701	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
4702
4703	if (fParavirtTsc)
4704	{
4705	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
4706	information before every VM-entry, hence disable it for performance sake. */
4707	#if 0
4708	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
4709	AssertRC(rc);
4710	#endif
4711	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatTscParavirt);
4712	}
4713
4714	if ( fOffsettedTsc
4715	&& RT_LIKELY(!pVCpu->hmr0.s.fDebugWantRdTscExit))
4716	{
4717	if (pVmxTransient->fIsNestedGuest)
4718	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
4719	vmxHCSetTscOffsetVmcs(pVCpu, pVmcsInfo, uTscOffset);
4720	vmxHCRemoveProcCtlsVmcs(pVCpu, pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
4721	}
4722	else
4723	{
4724	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4725	vmxHCSetProcCtlsVmcs(pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
4726	}
4727	}
4728	#endif /* !IN_RING0 */
4729
4730
4731	/**
4732	* Gets the IEM exception flags for the specified vector and IDT vectoring /
4733	* VM-exit interruption info type.
4734	*
4735	* @returns The IEM exception flags.
4736	* @param uVector The event vector.
4737	* @param uVmxEventType The VMX event type.
4738	*
4739	* @remarks This function currently only constructs flags required for
4740	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
4741	* and CR2 aspects of an exception are not included).
4742	*/
4743	static uint32_t vmxHCGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
4744	{
4745	uint32_t fIemXcptFlags;
4746	switch (uVmxEventType)
4747	{
4748	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
4749	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
4750	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
4751	break;
4752
4753	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
4754	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
4755	break;
4756
4757	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
4758	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
4759	break;
4760
4761	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
4762	{
4763	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
4764	if (uVector == X86_XCPT_BP)
4765	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
4766	else if (uVector == X86_XCPT_OF)
4767	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
4768	else
4769	{
4770	fIemXcptFlags = 0;
4771	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
4772	}
4773	break;
4774	}
4775
4776	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
4777	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
4778	break;
4779
4780	default:
4781	fIemXcptFlags = 0;
4782	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
4783	break;
4784	}
4785	return fIemXcptFlags;
4786	}
4787
4788
4789	/**
4790	* Sets an event as a pending event to be injected into the guest.
4791	*
4792	* @param pVCpu The cross context virtual CPU structure.
4793	* @param u32IntInfo The VM-entry interruption-information field.
4794	* @param cbInstr The VM-entry instruction length in bytes (for
4795	* software interrupts, exceptions and privileged
4796	* software exceptions).
4797	* @param u32ErrCode The VM-entry exception error code.
4798	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4799	* page-fault.
4800	*/
4801	DECLINLINE(void) vmxHCSetPendingEvent(PVMCPUCC pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4802	RTGCUINTPTR GCPtrFaultAddress)
4803	{
4804	Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending);
4805	VCPU_2_VMXSTATE(pVCpu).Event.fPending = true;
4806	VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo = u32IntInfo;
4807	VCPU_2_VMXSTATE(pVCpu).Event.u32ErrCode = u32ErrCode;
4808	VCPU_2_VMXSTATE(pVCpu).Event.cbInstr = cbInstr;
4809	VCPU_2_VMXSTATE(pVCpu).Event.GCPtrFaultAddress = GCPtrFaultAddress;
4810	}
4811
4812
4813	/**
4814	* Sets an external interrupt as pending-for-injection into the VM.
4815	*
4816	* @param pVCpu The cross context virtual CPU structure.
4817	* @param u8Interrupt The external interrupt vector.
4818	*/
4819	DECLINLINE(void) vmxHCSetPendingExtInt(PVMCPUCC pVCpu, uint8_t u8Interrupt)
4820	{
4821	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
4822	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
4823	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
4824	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
4825	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4826	}
4827
4828
4829	/**
4830	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
4831	*
4832	* @param pVCpu The cross context virtual CPU structure.
4833	*/
4834	DECLINLINE(void) vmxHCSetPendingXcptNmi(PVMCPUCC pVCpu)
4835	{
4836	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
4837	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
4838	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
4839	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
4840	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4841	}
4842
4843
4844	/**
4845	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
4846	*
4847	* @param pVCpu The cross context virtual CPU structure.
4848	*/
4849	DECLINLINE(void) vmxHCSetPendingXcptDF(PVMCPUCC pVCpu)
4850	{
4851	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
4852	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
4853	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
4854	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
4855	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4856	}
4857
4858
4859	/**
4860	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
4861	*
4862	* @param pVCpu The cross context virtual CPU structure.
4863	*/
4864	DECLINLINE(void) vmxHCSetPendingXcptUD(PVMCPUCC pVCpu)
4865	{
4866	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
4867	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
4868	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
4869	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
4870	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4871	}
4872
4873
4874	/**
4875	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
4876	*
4877	* @param pVCpu The cross context virtual CPU structure.
4878	*/
4879	DECLINLINE(void) vmxHCSetPendingXcptDB(PVMCPUCC pVCpu)
4880	{
4881	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
4882	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
4883	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
4884	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
4885	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4886	}
4887
4888
4889	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4890	/**
4891	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
4892	*
4893	* @param pVCpu The cross context virtual CPU structure.
4894	* @param u32ErrCode The error code for the general-protection exception.
4895	*/
4896	DECLINLINE(void) vmxHCSetPendingXcptGP(PVMCPUCC pVCpu, uint32_t u32ErrCode)
4897	{
4898	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
4899	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
4900	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
4901	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
4902	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
4903	}
4904
4905
4906	/**
4907	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
4908	*
4909	* @param pVCpu The cross context virtual CPU structure.
4910	* @param u32ErrCode The error code for the stack exception.
4911	*/
4912	DECLINLINE(void) vmxHCSetPendingXcptSS(PVMCPUCC pVCpu, uint32_t u32ErrCode)
4913	{
4914	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
4915	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
4916	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
4917	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
4918	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
4919	}
4920	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
4921
4922
4923	/**
4924	* Fixes up attributes for the specified segment register.
4925	*
4926	* @param pVCpu The cross context virtual CPU structure.
4927	* @param pSelReg The segment register that needs fixing.
4928	* @param pszRegName The register name (for logging and assertions).
4929	*/
4930	static void vmxHCFixUnusableSegRegAttr(PVMCPUCC pVCpu, PCPUMSELREG pSelReg, const char *pszRegName)
4931	{
4932	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
4933
4934	/*
4935	* If VT-x marks the segment as unusable, most other bits remain undefined:
4936	* - For CS the L, D and G bits have meaning.
4937	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
4938	* - For the remaining data segments no bits are defined.
4939	*
4940	* The present bit and the unusable bit has been observed to be set at the
4941	* same time (the selector was supposed to be invalid as we started executing
4942	* a V8086 interrupt in ring-0).
4943	*
4944	* What should be important for the rest of the VBox code, is that the P bit is
4945	* cleared. Some of the other VBox code recognizes the unusable bit, but
4946	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
4947	* safe side here, we'll strip off P and other bits we don't care about. If
4948	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
4949	*
4950	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
4951	*/
4952	#ifdef VBOX_STRICT
4953	uint32_t const uAttr = pSelReg->Attr.u;
4954	#endif
4955
4956	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
4957	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
4958	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
4959
4960	#ifdef VBOX_STRICT
4961	# ifdef IN_RING0
4962	VMMRZCallRing3Disable(pVCpu);
4963	# endif
4964	Log4Func(("Unusable %s: sel=%#x attr=%#x -> %#x\n", pszRegName, pSelReg->Sel, uAttr, pSelReg->Attr.u));
4965	# ifdef DEBUG_bird
4966	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
4967	("%s: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
4968	pszRegName, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
4969	# endif
4970	# ifdef IN_RING0
4971	VMMRZCallRing3Enable(pVCpu);
4972	# endif
4973	NOREF(uAttr);
4974	#endif
4975	RT_NOREF2(pVCpu, pszRegName);
4976	}
4977
4978
4979	/**
4980	* Imports a guest segment register from the current VMCS into the guest-CPU
4981	* context.
4982	*
4983	* @param pVCpu The cross context virtual CPU structure.
4984	* @param iSegReg The segment register number (X86_SREG_XXX).
4985	*
4986	* @remarks Called with interrupts and/or preemption disabled.
4987	*/
4988	static void vmxHCImportGuestSegReg(PVMCPUCC pVCpu, uint32_t iSegReg)
4989	{
4990	Assert(iSegReg < X86_SREG_COUNT);
4991	Assert((uint32_t)VMX_VMCS16_GUEST_SEG_SEL(iSegReg) == g_aVmcsSegSel[iSegReg]);
4992	Assert((uint32_t)VMX_VMCS32_GUEST_SEG_LIMIT(iSegReg) == g_aVmcsSegLimit[iSegReg]);
4993	Assert((uint32_t)VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(iSegReg) == g_aVmcsSegAttr[iSegReg]);
4994	Assert((uint32_t)VMX_VMCS_GUEST_SEG_BASE(iSegReg) == g_aVmcsSegBase[iSegReg]);
4995
4996	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
4997
4998	uint16_t u16Sel;
4999	int rc = VMX_VMCS_READ_16(pVCpu, VMX_VMCS16_GUEST_SEG_SEL(iSegReg), &u16Sel); AssertRC(rc);
5000	pSelReg->Sel = u16Sel;
5001	pSelReg->ValidSel = u16Sel;
5002
5003	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SEG_LIMIT(iSegReg), &pSelReg->u32Limit); AssertRC(rc);
5004	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SEG_BASE(iSegReg), &pSelReg->u64Base); AssertRC(rc);
5005
5006	uint32_t u32Attr;
5007	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(iSegReg), &u32Attr); AssertRC(rc);
5008	pSelReg->Attr.u = u32Attr;
5009	if (u32Attr & X86DESCATTR_UNUSABLE)
5010	vmxHCFixUnusableSegRegAttr(pVCpu, pSelReg, "ES\0CS\0SS\0DS\0FS\0GS" + iSegReg * 3);
5011
5012	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5013	}
5014
5015
5016	/**
5017	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
5018	*
5019	* @param pVCpu The cross context virtual CPU structure.
5020	*
5021	* @remarks Called with interrupts and/or preemption disabled.
5022	*/
5023	static void vmxHCImportGuestLdtr(PVMCPUCC pVCpu)
5024	{
5025	uint16_t u16Sel;
5026	uint64_t u64Base;
5027	uint32_t u32Limit, u32Attr;
5028	int rc = VMX_VMCS_READ_16(pVCpu, VMX_VMCS16_GUEST_LDTR_SEL, &u16Sel); AssertRC(rc);
5029	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit); AssertRC(rc);
5030	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc);
5031	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_LDTR_BASE, &u64Base); AssertRC(rc);
5032
5033	pVCpu->cpum.GstCtx.ldtr.Sel = u16Sel;
5034	pVCpu->cpum.GstCtx.ldtr.ValidSel = u16Sel;
5035	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
5036	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
5037	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
5038	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
5039	if (u32Attr & X86DESCATTR_UNUSABLE)
5040	vmxHCFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, "LDTR");
5041	}
5042
5043
5044	/**
5045	* Imports the guest TR from the current VMCS into the guest-CPU context.
5046	*
5047	* @param pVCpu The cross context virtual CPU structure.
5048	*
5049	* @remarks Called with interrupts and/or preemption disabled.
5050	*/
5051	static void vmxHCImportGuestTr(PVMCPUCC pVCpu)
5052	{
5053	uint16_t u16Sel;
5054	uint64_t u64Base;
5055	uint32_t u32Limit, u32Attr;
5056	int rc = VMX_VMCS_READ_16(pVCpu, VMX_VMCS16_GUEST_TR_SEL, &u16Sel); AssertRC(rc);
5057	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit); AssertRC(rc);
5058	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc);
5059	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_TR_BASE, &u64Base); AssertRC(rc);
5060
5061	pVCpu->cpum.GstCtx.tr.Sel = u16Sel;
5062	pVCpu->cpum.GstCtx.tr.ValidSel = u16Sel;
5063	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
5064	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
5065	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
5066	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
5067	/* TR is the only selector that can never be unusable. */
5068	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
5069	}
5070
5071
5072	/**
5073	* Imports the guest RIP from the VMCS back into the guest-CPU context.
5074	*
5075	* @param pVCpu The cross context virtual CPU structure.
5076	*
5077	* @remarks Called with interrupts and/or preemption disabled, should not assert!
5078	* @remarks Do -not- call this function directly, use vmxHCImportGuestState()
5079	* instead!!!
5080	*/
5081	static void vmxHCImportGuestRip(PVMCPUCC pVCpu)
5082	{
5083	uint64_t u64Val;
5084	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5085	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
5086	{
5087	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RIP, &u64Val);
5088	AssertRC(rc);
5089
5090	pCtx->rip = u64Val;
5091	EMHistoryUpdatePC(pVCpu, pCtx->rip, false);
5092	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
5093	}
5094	}
5095
5096
5097	/**
5098	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
5099	*
5100	* @param pVCpu The cross context virtual CPU structure.
5101	* @param pVmcsInfo The VMCS info. object.
5102	*
5103	* @remarks Called with interrupts and/or preemption disabled, should not assert!
5104	* @remarks Do -not- call this function directly, use vmxHCImportGuestState()
5105	* instead!!!
5106	*/
5107	static void vmxHCImportGuestRFlags(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
5108	{
5109	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5110	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
5111	{
5112	uint64_t u64Val;
5113	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RFLAGS, &u64Val);
5114	AssertRC(rc);
5115
5116	pCtx->rflags.u64 = u64Val;
5117	#ifdef IN_RING0
5118	PCVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
5119	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
5120	{
5121	pCtx->eflags.Bits.u1VM = 0;
5122	pCtx->eflags.Bits.u2IOPL = pVmcsInfoShared->RealMode.Eflags.Bits.u2IOPL;
5123	}
5124	#else
5125	RT_NOREF(pVmcsInfo);
5126	#endif
5127	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
5128	}
5129	}
5130
5131
5132	/**
5133	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
5134	* context.
5135	*
5136	* @param pVCpu The cross context virtual CPU structure.
5137	* @param pVmcsInfo The VMCS info. object.
5138	*
5139	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
5140	* do not log!
5141	* @remarks Do -not- call this function directly, use vmxHCImportGuestState()
5142	* instead!!!
5143	*/
5144	static void vmxHCImportGuestIntrState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
5145	{
5146	uint32_t u32Val;
5147	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &u32Val); AssertRC(rc);
5148	if (!u32Val)
5149	{
5150	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5151	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5152	CPUMSetGuestNmiBlocking(pVCpu, false);
5153	}
5154	else
5155	{
5156	/*
5157	* We must import RIP here to set our EM interrupt-inhibited state.
5158	* We also import RFLAGS as our code that evaluates pending interrupts
5159	* before VM-entry requires it.
5160	*/
5161	vmxHCImportGuestRip(pVCpu);
5162	vmxHCImportGuestRFlags(pVCpu, pVmcsInfo);
5163
5164	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
5165	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
5166	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5167	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5168
5169	bool const fNmiBlocking = RT_BOOL(u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
5170	CPUMSetGuestNmiBlocking(pVCpu, fNmiBlocking);
5171	}
5172	}
5173
5174
5175	/**
5176	* Worker for VMXR0ImportStateOnDemand.
5177	*
5178	* @returns VBox status code.
5179	* @param pVCpu The cross context virtual CPU structure.
5180	* @param pVmcsInfo The VMCS info. object.
5181	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
5182	*/
5183	static int vmxHCImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
5184	{
5185	int rc = VINF_SUCCESS;
5186	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5187	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5188	uint32_t u32Val;
5189
5190	/*
5191	* Note! This is hack to workaround a mysterious BSOD observed with release builds
5192	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
5193	* neither are other host platforms.
5194	*
5195	* Committing this temporarily as it prevents BSOD.
5196	*
5197	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
5198	*/
5199	# ifdef RT_OS_WINDOWS
5200	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
5201	return VERR_HM_IPE_1;
5202	# endif
5203
5204	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x);
5205
5206	#ifdef IN_RING0
5207	/*
5208	* We disable interrupts to make the updating of the state and in particular
5209	* the fExtrn modification atomic wrt to preemption hooks.
5210	*/
5211	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
5212	#endif
5213
5214	fWhat &= pCtx->fExtrn;
5215	if (fWhat)
5216	{
5217	do
5218	{
5219	if (fWhat & CPUMCTX_EXTRN_RIP)
5220	vmxHCImportGuestRip(pVCpu);
5221
5222	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
5223	vmxHCImportGuestRFlags(pVCpu, pVmcsInfo);
5224
5225	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
5226	vmxHCImportGuestIntrState(pVCpu, pVmcsInfo);
5227
5228	if (fWhat & CPUMCTX_EXTRN_RSP)
5229	{
5230	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RSP, &pCtx->rsp);
5231	AssertRC(rc);
5232	}
5233
5234	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
5235	{
5236	PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
5237	#ifndef IN_NEM_DARWIN
5238	bool const fRealOnV86Active = pVmcsInfoShared->RealMode.fRealOnV86Active;
5239	#else
5240	bool const fRealOnV86Active = false; /* HV supports only unrestricted guest execution. */
5241	#endif
5242	if (fWhat & CPUMCTX_EXTRN_CS)
5243	{
5244	vmxHCImportGuestSegReg(pVCpu, X86_SREG_CS);
5245	vmxHCImportGuestRip(pVCpu);
5246	if (fRealOnV86Active)
5247	pCtx->cs.Attr.u = pVmcsInfoShared->RealMode.AttrCS.u;
5248	EMHistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
5249	}
5250	if (fWhat & CPUMCTX_EXTRN_SS)
5251	{
5252	vmxHCImportGuestSegReg(pVCpu, X86_SREG_SS);
5253	if (fRealOnV86Active)
5254	pCtx->ss.Attr.u = pVmcsInfoShared->RealMode.AttrSS.u;
5255	}
5256	if (fWhat & CPUMCTX_EXTRN_DS)
5257	{
5258	vmxHCImportGuestSegReg(pVCpu, X86_SREG_DS);
5259	if (fRealOnV86Active)
5260	pCtx->ds.Attr.u = pVmcsInfoShared->RealMode.AttrDS.u;
5261	}
5262	if (fWhat & CPUMCTX_EXTRN_ES)
5263	{
5264	vmxHCImportGuestSegReg(pVCpu, X86_SREG_ES);
5265	if (fRealOnV86Active)
5266	pCtx->es.Attr.u = pVmcsInfoShared->RealMode.AttrES.u;
5267	}
5268	if (fWhat & CPUMCTX_EXTRN_FS)
5269	{
5270	vmxHCImportGuestSegReg(pVCpu, X86_SREG_FS);
5271	if (fRealOnV86Active)
5272	pCtx->fs.Attr.u = pVmcsInfoShared->RealMode.AttrFS.u;
5273	}
5274	if (fWhat & CPUMCTX_EXTRN_GS)
5275	{
5276	vmxHCImportGuestSegReg(pVCpu, X86_SREG_GS);
5277	if (fRealOnV86Active)
5278	pCtx->gs.Attr.u = pVmcsInfoShared->RealMode.AttrGS.u;
5279	}
5280	}
5281
5282	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
5283	{
5284	if (fWhat & CPUMCTX_EXTRN_LDTR)
5285	vmxHCImportGuestLdtr(pVCpu);
5286
5287	if (fWhat & CPUMCTX_EXTRN_GDTR)
5288	{
5289	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE, &pCtx->gdtr.pGdt); AssertRC(rc);
5290	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRC(rc);
5291	pCtx->gdtr.cbGdt = u32Val;
5292	}
5293
5294	/* Guest IDTR. */
5295	if (fWhat & CPUMCTX_EXTRN_IDTR)
5296	{
5297	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE, &pCtx->idtr.pIdt); AssertRC(rc);
5298	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRC(rc);
5299	pCtx->idtr.cbIdt = u32Val;
5300	}
5301
5302	/* Guest TR. */
5303	if (fWhat & CPUMCTX_EXTRN_TR)
5304	{
5305	#ifdef IN_RING0
5306	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
5307	don't need to import that one. */
5308	if (!pVmcsInfo->pShared->RealMode.fRealOnV86Active)
5309	#endif
5310	vmxHCImportGuestTr(pVCpu);
5311	}
5312	}
5313
5314	if (fWhat & CPUMCTX_EXTRN_DR7)
5315	{
5316	#ifdef IN_RING0
5317	if (!pVCpu->hmr0.s.fUsingHyperDR7)
5318	#endif
5319	{
5320	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_DR7, &pCtx->dr[7]);
5321	AssertRC(rc);
5322	}
5323	}
5324
5325	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
5326	{
5327	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip); AssertRC(rc);
5328	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp); AssertRC(rc);
5329	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRC(rc);
5330	pCtx->SysEnter.cs = u32Val;
5331	}
5332
5333	#ifdef IN_RING0
5334	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
5335	{
5336	if ( pVM->hmr0.s.fAllow64BitGuests
5337	&& (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
5338	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
5339	}
5340
5341	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
5342	{
5343	if ( pVM->hmr0.s.fAllow64BitGuests
5344	&& (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
5345	{
5346	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
5347	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
5348	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
5349	}
5350	}
5351
5352	if (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
5353	{
5354	PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
5355	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
5356	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
5357	Assert(pMsrs);
5358	Assert(cMsrs <= VMX_MISC_MAX_MSRS(g_HmMsrs.u.vmx.u64Misc));
5359	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
5360	for (uint32_t i = 0; i < cMsrs; i++)
5361	{
5362	uint32_t const idMsr = pMsrs[i].u32Msr;
5363	switch (idMsr)
5364	{
5365	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
5366	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
5367	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
5368	default:
5369	{
5370	uint32_t idxLbrMsr;
5371	if (VM_IS_VMX_LBR(pVM))
5372	{
5373	if (vmxHCIsLbrBranchFromMsr(pVM, idMsr, &idxLbrMsr))
5374	{
5375	Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr));
5376	pVmcsInfoShared->au64LbrFromIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
5377	break;
5378	}
5379	if (vmxHCIsLbrBranchToMsr(pVM, idMsr, &idxLbrMsr))
5380	{
5381	Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr));
5382	pVmcsInfoShared->au64LbrToIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
5383	break;
5384	}
5385	if (idMsr == pVM->hmr0.s.vmx.idLbrTosMsr)
5386	{
5387	pVmcsInfoShared->u64LbrTosMsr = pMsrs[i].u64Value;
5388	break;
5389	}
5390	/* Fallthru (no break) */
5391	}
5392	pCtx->fExtrn = 0;
5393	VCPU_2_VMXSTATE(pVCpu).u32HMError = pMsrs->u32Msr;
5394	ASMSetFlags(fEFlags);
5395	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
5396	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5397	}
5398	}
5399	}
5400	}
5401	#endif
5402
5403	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
5404	{
5405	if (fWhat & CPUMCTX_EXTRN_CR0)
5406	{
5407	uint64_t u64Cr0;
5408	uint64_t u64Shadow;
5409	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0, &u64Cr0); AssertRC(rc);
5410	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow); AssertRC(rc);
5411	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
5412	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
5413	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
5414	#else
5415	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
5416	{
5417	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
5418	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
5419	}
5420	else
5421	{
5422	/*
5423	* We've merged the guest and nested-guest's CR0 guest/host mask while executing
5424	* the nested-guest using hardware-assisted VMX. Accordingly we need to
5425	* re-construct CR0. See @bugref{9180#c95} for details.
5426	*/
5427	PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo;
5428	PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
5429	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
5430	\| (pVmcsNstGst->u64GuestCr0.u & pVmcsNstGst->u64Cr0Mask.u)
5431	\| (u64Shadow & (pVmcsInfoGst->u64Cr0Mask & ~pVmcsNstGst->u64Cr0Mask.u));
5432	}
5433	#endif
5434	#ifdef IN_RING0
5435	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
5436	#endif
5437	CPUMSetGuestCR0(pVCpu, u64Cr0);
5438	#ifdef IN_RING0
5439	VMMRZCallRing3Enable(pVCpu);
5440	#endif
5441	}
5442
5443	if (fWhat & CPUMCTX_EXTRN_CR4)
5444	{
5445	uint64_t u64Cr4;
5446	uint64_t u64Shadow;
5447	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR4, &u64Cr4); AssertRC(rc);
5448	rc \|= VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow); AssertRC(rc);
5449	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
5450	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
5451	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
5452	#else
5453	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
5454	{
5455	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
5456	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
5457	}
5458	else
5459	{
5460	/*
5461	* We've merged the guest and nested-guest's CR4 guest/host mask while executing
5462	* the nested-guest using hardware-assisted VMX. Accordingly we need to
5463	* re-construct CR4. See @bugref{9180#c95} for details.
5464	*/
5465	PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo;
5466	PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
5467	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
5468	\| (pVmcsNstGst->u64GuestCr4.u & pVmcsNstGst->u64Cr4Mask.u)
5469	\| (u64Shadow & (pVmcsInfoGst->u64Cr4Mask & ~pVmcsNstGst->u64Cr4Mask.u));
5470	}
5471	#endif
5472	pCtx->cr4 = u64Cr4;
5473	}
5474
5475	if (fWhat & CPUMCTX_EXTRN_CR3)
5476	{
5477	/* CR0.PG bit changes are always intercepted, so it's up to date. */
5478	if ( VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
5479	\|\| ( VM_IS_VMX_NESTED_PAGING(pVM)
5480	&& CPUMIsGuestPagingEnabledEx(pCtx)))
5481	{
5482	uint64_t u64Cr3;
5483	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR3, &u64Cr3); AssertRC(rc);
5484	if (pCtx->cr3 != u64Cr3)
5485	{
5486	pCtx->cr3 = u64Cr3;
5487	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5488	}
5489
5490	/*
5491	* If the guest is in PAE mode, sync back the PDPE's into the guest state.
5492	* CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date.
5493	*/
5494	if (CPUMIsGuestInPAEModeEx(pCtx))
5495	{
5496	X86PDPE aPaePdpes[4];
5497	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, &aPaePdpes[0].u); AssertRC(rc);
5498	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, &aPaePdpes[1].u); AssertRC(rc);
5499	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, &aPaePdpes[2].u); AssertRC(rc);
5500	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, &aPaePdpes[3].u); AssertRC(rc);
5501	if (memcmp(&aPaePdpes[0], &pCtx->aPaePdpes[0], sizeof(aPaePdpes)))
5502	{
5503	memcpy(&pCtx->aPaePdpes[0], &aPaePdpes[0], sizeof(aPaePdpes));
5504	/* PGM now updates PAE PDPTEs while updating CR3. */
5505	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5506	}
5507	}
5508	}
5509	}
5510	}
5511
5512	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5513	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
5514	{
5515	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
5516	&& !CPUMIsGuestInVmxNonRootMode(pCtx))
5517	{
5518	Assert(CPUMIsGuestInVmxRootMode(pCtx));
5519	rc = vmxHCCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
5520	if (RT_SUCCESS(rc))
5521	{ /* likely */ }
5522	else
5523	break;
5524	}
5525	}
5526	#endif
5527	} while (0);
5528
5529	if (RT_SUCCESS(rc))
5530	{
5531	/* Update fExtrn. */
5532	pCtx->fExtrn &= ~fWhat;
5533
5534	/* If everything has been imported, clear the HM keeper bit. */
5535	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
5536	{
5537	#ifndef IN_NEM_DARWIN
5538	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
5539	#else
5540	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_NEM;
5541	#endif
5542	Assert(!pCtx->fExtrn);
5543	}
5544	}
5545	}
5546	#ifdef IN_RING0
5547	else
5548	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
5549
5550	/*
5551	* Restore interrupts.
5552	*/
5553	ASMSetFlags(fEFlags);
5554	#endif
5555
5556	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x);
5557
5558	if (RT_SUCCESS(rc))
5559	{ /* likely */ }
5560	else
5561	return rc;
5562
5563	/*
5564	* Honor any pending CR3 updates.
5565	*
5566	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> VMXR0CallRing3Callback()
5567	* -> VMMRZCallRing3Disable() -> vmxHCImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5568	* -> continue with VM-exit handling -> vmxHCImportGuestState() and here we are.
5569	*
5570	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
5571	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
5572	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
5573	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
5574	*
5575	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5576	*
5577	* The force-flag is checked first as it's cheaper for potential superfluous calls to this function.
5578	*/
5579	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3)
5580	#ifdef IN_RING0
5581	&& VMMRZCallRing3IsEnabled(pVCpu)
5582	#endif
5583	)
5584	{
5585	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
5586	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu), false /* fPdpesMapped */);
5587	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5588	}
5589
5590	return VINF_SUCCESS;
5591	}
5592
5593
5594	/**
5595	* Check per-VM and per-VCPU force flag actions that require us to go back to
5596	* ring-3 for one reason or another.
5597	*
5598	* @returns Strict VBox status code (i.e. informational status codes too)
5599	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5600	* ring-3.
5601	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5602	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5603	* interrupts)
5604	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5605	* all EMTs to be in ring-3.
5606	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5607	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5608	* to the EM loop.
5609	*
5610	* @param pVCpu The cross context virtual CPU structure.
5611	* @param fIsNestedGuest Flag whether this is for a for a pending nested guest event.
5612	* @param fStepping Whether we are single-stepping the guest using the
5613	* hypervisor debugger.
5614	*
5615	* @remarks This might cause nested-guest VM-exits, caller must check if the guest
5616	* is no longer in VMX non-root mode.
5617	*/
5618	static VBOXSTRICTRC vmxHCCheckForceFlags(PVMCPUCC pVCpu, bool fIsNestedGuest, bool fStepping)
5619	{
5620	#ifdef IN_RING0
5621	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5622	#endif
5623
5624	/*
5625	* Update pending interrupts into the APIC's IRR.
5626	*/
5627	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
5628	APICUpdatePendingInterrupts(pVCpu);
5629
5630	/*
5631	* Anything pending? Should be more likely than not if we're doing a good job.
5632	*/
5633	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5634	if ( !fStepping
5635	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
5636	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
5637	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
5638	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
5639	return VINF_SUCCESS;
5640
5641	/* Pending PGM C3 sync. */
5642	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5643	{
5644	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5645	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
5646	VBOXSTRICTRC rcStrict = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
5647	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5648	if (rcStrict != VINF_SUCCESS)
5649	{
5650	AssertRC(VBOXSTRICTRC_VAL(rcStrict));
5651	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
5652	return rcStrict;
5653	}
5654	}
5655
5656	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5657	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
5658	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5659	{
5660	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchHmToR3FF);
5661	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
5662	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5663	return rc;
5664	}
5665
5666	/* Pending VM request packets, such as hardware interrupts. */
5667	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
5668	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
5669	{
5670	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchVmReq);
5671	Log4Func(("Pending VM request forcing us back to ring-3\n"));
5672	return VINF_EM_PENDING_REQUEST;
5673	}
5674
5675	/* Pending PGM pool flushes. */
5676	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5677	{
5678	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchPgmPoolFlush);
5679	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
5680	return VINF_PGM_POOL_FLUSH_PENDING;
5681	}
5682
5683	/* Pending DMA requests. */
5684	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
5685	{
5686	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchDma);
5687	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
5688	return VINF_EM_RAW_TO_R3;
5689	}
5690
5691	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5692	/*
5693	* Pending nested-guest events.
5694	*
5695	* Please note the priority of these events are specified and important.
5696	* See Intel spec. 29.4.3.2 "APIC-Write Emulation".
5697	* See Intel spec. 6.9 "Priority Among Simultaneous Exceptions And Interrupts".
5698	*/
5699	if (fIsNestedGuest)
5700	{
5701	/* Pending nested-guest APIC-write. */
5702	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
5703	{
5704	Log4Func(("Pending nested-guest APIC-write\n"));
5705	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitApicWrite(pVCpu);
5706	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
5707	return rcStrict;
5708	}
5709
5710	/* Pending nested-guest monitor-trap flag (MTF). */
5711	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF))
5712	{
5713	Log4Func(("Pending nested-guest MTF\n"));
5714	VBOXSTRICTRC rcStrict = IEMExecVmxVmexit(pVCpu, VMX_EXIT_MTF, 0 /* uExitQual */);
5715	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
5716	return rcStrict;
5717	}
5718
5719	/* Pending nested-guest VMX-preemption timer expired. */
5720	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER))
5721	{
5722	Log4Func(("Pending nested-guest preempt timer\n"));
5723	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitPreemptTimer(pVCpu);
5724	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
5725	return rcStrict;
5726	}
5727	}
5728	#else
5729	NOREF(fIsNestedGuest);
5730	#endif
5731
5732	return VINF_SUCCESS;
5733	}
5734
5735
5736	/**
5737	* Converts any TRPM trap into a pending HM event. This is typically used when
5738	* entering from ring-3 (not longjmp returns).
5739	*
5740	* @param pVCpu The cross context virtual CPU structure.
5741	*/
5742	static void vmxHCTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
5743	{
5744	Assert(TRPMHasTrap(pVCpu));
5745	Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending);
5746
5747	uint8_t uVector;
5748	TRPMEVENT enmTrpmEvent;
5749	uint32_t uErrCode;
5750	RTGCUINTPTR GCPtrFaultAddress;
5751	uint8_t cbInstr;
5752	bool fIcebp;
5753
5754	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, &fIcebp);
5755	AssertRC(rc);
5756
5757	uint32_t u32IntInfo;
5758	u32IntInfo = uVector \| VMX_IDT_VECTORING_INFO_VALID;
5759	u32IntInfo \|= HMTrpmEventTypeToVmxEventType(uVector, enmTrpmEvent, fIcebp);
5760
5761	rc = TRPMResetTrap(pVCpu);
5762	AssertRC(rc);
5763	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5764	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5765
5766	vmxHCSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5767	}
5768
5769
5770	/**
5771	* Converts the pending HM event into a TRPM trap.
5772	*
5773	* @param pVCpu The cross context virtual CPU structure.
5774	*/
5775	static void vmxHCPendingEventToTrpmTrap(PVMCPUCC pVCpu)
5776	{
5777	Assert(VCPU_2_VMXSTATE(pVCpu).Event.fPending);
5778
5779	/* If a trap was already pending, we did something wrong! */
5780	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5781
5782	uint32_t const u32IntInfo = VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo;
5783	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(u32IntInfo);
5784	TRPMEVENT const enmTrapType = HMVmxEventTypeToTrpmEventType(u32IntInfo);
5785
5786	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5787
5788	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5789	AssertRC(rc);
5790
5791	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
5792	TRPMSetErrorCode(pVCpu, VCPU_2_VMXSTATE(pVCpu).Event.u32ErrCode);
5793
5794	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(u32IntInfo))
5795	TRPMSetFaultAddress(pVCpu, VCPU_2_VMXSTATE(pVCpu).Event.GCPtrFaultAddress);
5796	else
5797	{
5798	uint8_t const uVectorType = VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo);
5799	switch (uVectorType)
5800	{
5801	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5802	TRPMSetTrapDueToIcebp(pVCpu);
5803	RT_FALL_THRU();
5804	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5805	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5806	{
5807	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5808	\|\| ( uVector == X86_XCPT_BP /* INT3 */
5809	\|\| uVector == X86_XCPT_OF /* INTO */
5810	\|\| uVector == X86_XCPT_DB /* INT1 (ICEBP) */),
5811	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5812	TRPMSetInstrLength(pVCpu, VCPU_2_VMXSTATE(pVCpu).Event.cbInstr);
5813	break;
5814	}
5815	}
5816	}
5817
5818	/* We're now done converting the pending event. */
5819	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false;
5820	}
5821
5822
5823	/**
5824	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
5825	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
5826	*
5827	* @param pVCpu The cross context virtual CPU structure.
5828	* @param pVmcsInfo The VMCS info. object.
5829	*/
5830	static void vmxHCSetIntWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
5831	{
5832	if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
5833	{
5834	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
5835	{
5836	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
5837	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
5838	AssertRC(rc);
5839	}
5840	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
5841	}
5842
5843
5844	/**
5845	* Clears the interrupt-window exiting control in the VMCS.
5846	*
5847	* @param pVCpu The cross context virtual CPU structure.
5848	* @param pVmcsInfo The VMCS info. object.
5849	*/
5850	DECLINLINE(void) vmxHCClearIntWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
5851	{
5852	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
5853	{
5854	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
5855	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
5856	AssertRC(rc);
5857	}
5858	}
5859
5860
5861	/**
5862	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
5863	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
5864	*
5865	* @param pVCpu The cross context virtual CPU structure.
5866	* @param pVmcsInfo The VMCS info. object.
5867	*/
5868	static void vmxHCSetNmiWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
5869	{
5870	if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
5871	{
5872	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
5873	{
5874	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
5875	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
5876	AssertRC(rc);
5877	Log4Func(("Setup NMI-window exiting\n"));
5878	}
5879	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
5880	}
5881
5882
5883	/**
5884	* Clears the NMI-window exiting control in the VMCS.
5885	*
5886	* @param pVCpu The cross context virtual CPU structure.
5887	* @param pVmcsInfo The VMCS info. object.
5888	*/
5889	DECLINLINE(void) vmxHCClearNmiWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
5890	{
5891	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
5892	{
5893	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
5894	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
5895	AssertRC(rc);
5896	}
5897	}
5898
5899
5900	#ifdef IN_RING0
5901	/**
5902	* Does the necessary state syncing before returning to ring-3 for any reason
5903	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
5904	*
5905	* @returns VBox status code.
5906	* @param pVCpu The cross context virtual CPU structure.
5907	* @param fImportState Whether to import the guest state from the VMCS back
5908	* to the guest-CPU context.
5909	*
5910	* @remarks No-long-jmp zone!!!
5911	*/
5912	static int vmxHCLeave(PVMCPUCC pVCpu, bool fImportState)
5913	{
5914	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5915	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5916
5917	RTCPUID const idCpu = RTMpCpuId();
5918	Log4Func(("HostCpuId=%u\n", idCpu));
5919
5920	/*
5921	* !!! IMPORTANT !!!
5922	* If you modify code here, check whether VMXR0CallRing3Callback() needs to be updated too.
5923	*/
5924
5925	/* Save the guest state if necessary. */
5926	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
5927	if (fImportState)
5928	{
5929	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
5930	AssertRCReturn(rc, rc);
5931	}
5932
5933	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
5934	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
5935	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5936
5937	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
5938	#ifdef VBOX_STRICT
5939	if (CPUMIsHyperDebugStateActive(pVCpu))
5940	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
5941	#endif
5942	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
5943	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5944	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5945
5946	/* Restore host-state bits that VT-x only restores partially. */
5947	if (pVCpu->hmr0.s.vmx.fRestoreHostFlags > VMX_RESTORE_HOST_REQUIRED)
5948	{
5949	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hmr0.s.vmx.fRestoreHostFlags, idCpu));
5950	VMXRestoreHostState(pVCpu->hmr0.s.vmx.fRestoreHostFlags, &pVCpu->hmr0.s.vmx.RestoreHost);
5951	}
5952	pVCpu->hmr0.s.vmx.fRestoreHostFlags = 0;
5953
5954	/* Restore the lazy host MSRs as we're leaving VT-x context. */
5955	if (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
5956	{
5957	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
5958	if (!fImportState)
5959	{
5960	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
5961	AssertRCReturn(rc, rc);
5962	}
5963	vmxHCLazyRestoreHostMsrs(pVCpu);
5964	Assert(!pVCpu->hmr0.s.vmx.fLazyMsrs);
5965	}
5966	else
5967	pVCpu->hmr0.s.vmx.fLazyMsrs = 0;
5968
5969	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
5970	pVCpu->hmr0.s.vmx.fUpdatedHostAutoMsrs = false;
5971
5972	STAM_PROFILE_ADV_SET_STOPPED(&VCPU_2_VMXSTATS(pVCpu).StatEntry);
5973	STAM_PROFILE_ADV_SET_STOPPED(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState);
5974	STAM_PROFILE_ADV_SET_STOPPED(&VCPU_2_VMXSTATS(pVCpu).StatExportGuestState);
5975	STAM_PROFILE_ADV_SET_STOPPED(&VCPU_2_VMXSTATS(pVCpu).StatPreExit);
5976	STAM_PROFILE_ADV_SET_STOPPED(&VCPU_2_VMXSTATS(pVCpu).StatExitHandling);
5977	STAM_PROFILE_ADV_SET_STOPPED(&VCPU_2_VMXSTATS(pVCpu).StatExitIO);
5978	STAM_PROFILE_ADV_SET_STOPPED(&VCPU_2_VMXSTATS(pVCpu).StatExitMovCRx);
5979	STAM_PROFILE_ADV_SET_STOPPED(&VCPU_2_VMXSTATS(pVCpu).StatExitXcptNmi);
5980	STAM_PROFILE_ADV_SET_STOPPED(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry);
5981	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchLongJmpToR3);
5982
5983	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
5984
5985	/** @todo This partially defeats the purpose of having preemption hooks.
5986	* The problem is, deregistering the hooks should be moved to a place that
5987	* lasts until the EMT is about to be destroyed not everytime while leaving HM
5988	* context.
5989	*/
5990	int rc = vmxHCClearVmcs(pVmcsInfo);
5991	AssertRCReturn(rc, rc);
5992
5993	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5994	/*
5995	* A valid shadow VMCS is made active as part of VM-entry. It is necessary to
5996	* clear a shadow VMCS before allowing that VMCS to become active on another
5997	* logical processor. We may or may not be importing guest state which clears
5998	* it, so cover for it here.
5999	*
6000	* See Intel spec. 24.11.1 "Software Use of Virtual-Machine Control Structures".
6001	*/
6002	if ( pVmcsInfo->pvShadowVmcs
6003	&& pVmcsInfo->fShadowVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
6004	{
6005	rc = vmxHCClearShadowVmcs(pVmcsInfo);
6006	AssertRCReturn(rc, rc);
6007	}
6008
6009	/*
6010	* Flag that we need to re-export the host state if we switch to this VMCS before
6011	* executing guest or nested-guest code.
6012	*/
6013	pVmcsInfo->idHostCpuState = NIL_RTCPUID;
6014	#endif
6015
6016	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6017	NOREF(idCpu);
6018	return VINF_SUCCESS;
6019	}
6020
6021
6022	/**
6023	* Leaves the VT-x session.
6024	*
6025	* @returns VBox status code.
6026	* @param pVCpu The cross context virtual CPU structure.
6027	*
6028	* @remarks No-long-jmp zone!!!
6029	*/
6030	static int vmxHCLeaveSession(PVMCPUCC pVCpu)
6031	{
6032	HM_DISABLE_PREEMPT(pVCpu);
6033	HMVMX_ASSERT_CPU_SAFE(pVCpu);
6034	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6035	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6036
6037	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6038	and done this from the VMXR0ThreadCtxCallback(). */
6039	if (!pVCpu->hmr0.s.fLeaveDone)
6040	{
6041	int rc2 = vmxHCLeave(pVCpu, true /* fImportState */);
6042	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
6043	pVCpu->hmr0.s.fLeaveDone = true;
6044	}
6045	Assert(!pVCpu->cpum.GstCtx.fExtrn);
6046
6047	/*
6048	* !!! IMPORTANT !!!
6049	* If you modify code here, make sure to check whether VMXR0CallRing3Callback() needs to be updated too.
6050	*/
6051
6052	/* Deregister hook now that we've left HM context before re-enabling preemption. */
6053	/** @todo Deregistering here means we need to VMCLEAR always
6054	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
6055	* for calling VMMR0ThreadCtxHookDisable here! */
6056	VMMR0ThreadCtxHookDisable(pVCpu);
6057
6058	/* Leave HM context. This takes care of local init (term) and deregistering the longjmp-to-ring-3 callback. */
6059	int rc = HMR0LeaveCpu(pVCpu);
6060	HM_RESTORE_PREEMPT();
6061	return rc;
6062	}
6063
6064
6065	/**
6066	* Does the necessary state syncing before doing a longjmp to ring-3.
6067	*
6068	* @returns VBox status code.
6069	* @param pVCpu The cross context virtual CPU structure.
6070	*
6071	* @remarks No-long-jmp zone!!!
6072	*/
6073	DECLINLINE(int) vmxHCLongJmpToRing3(PVMCPUCC pVCpu)
6074	{
6075	return vmxHCLeaveSession(pVCpu);
6076	}
6077
6078
6079	/**
6080	* Take necessary actions before going back to ring-3.
6081	*
6082	* An action requires us to go back to ring-3. This function does the necessary
6083	* steps before we can safely return to ring-3. This is not the same as longjmps
6084	* to ring-3, this is voluntary and prepares the guest so it may continue
6085	* executing outside HM (recompiler/IEM).
6086	*
6087	* @returns VBox status code.
6088	* @param pVCpu The cross context virtual CPU structure.
6089	* @param rcExit The reason for exiting to ring-3. Can be
6090	* VINF_VMM_UNKNOWN_RING3_CALL.
6091	*/
6092	static int vmxHCExitToRing3(PVMCPUCC pVCpu, VBOXSTRICTRC rcExit)
6093	{
6094	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6095
6096	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6097	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6098	{
6099	VMXGetCurrentVmcs(&VCPU_2_VMXSTATE(pVCpu).vmx.LastError.HCPhysCurrentVmcs);
6100	VCPU_2_VMXSTATE(pVCpu).vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
6101	VCPU_2_VMXSTATE(pVCpu).vmx.LastError.idEnteredCpu = pVCpu->hmr0.s.idEnteredCpu;
6102	/* LastError.idCurrentCpu was updated in vmxHCPreRunGuestCommitted(). */
6103	}
6104
6105	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6106	VMMRZCallRing3Disable(pVCpu);
6107	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
6108
6109	/*
6110	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
6111	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
6112	*
6113	* This is because execution may continue from ring-3 and we would need to inject
6114	* the event from there (hence place it back in TRPM).
6115	*/
6116	if (VCPU_2_VMXSTATE(pVCpu).Event.fPending)
6117	{
6118	vmxHCPendingEventToTrpmTrap(pVCpu);
6119	Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending);
6120
6121	/* Clear the events from the VMCS. */
6122	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6123	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, 0); AssertRC(rc);
6124	}
6125	#ifdef VBOX_STRICT
6126	/*
6127	* We check for rcExit here since for errors like VERR_VMX_UNABLE_TO_START_VM (which are
6128	* fatal), we don't care about verifying duplicate injection of events. Errors like
6129	* VERR_EM_INTERPRET are converted to their VINF_* counterparts -prior- to calling this
6130	* function so those should and will be checked below.
6131	*/
6132	else if (RT_SUCCESS(rcExit))
6133	{
6134	/*
6135	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
6136	* This can be pretty hard to debug otherwise, interrupts might get injected twice
6137	* occasionally, see @bugref{9180#c42}.
6138	*
6139	* However, if the VM-entry failed, any VM entry-interruption info. field would
6140	* be left unmodified as the event would not have been injected to the guest. In
6141	* such cases, don't assert, we're not going to continue guest execution anyway.
6142	*/
6143	uint32_t uExitReason;
6144	uint32_t uEntryIntInfo;
6145	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6146	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
6147	AssertRC(rc);
6148	AssertMsg(VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason) \|\| !VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo),
6149	("uExitReason=%#RX32 uEntryIntInfo=%#RX32 rcExit=%d\n", uExitReason, uEntryIntInfo, VBOXSTRICTRC_VAL(rcExit)));
6150	}
6151	#endif
6152
6153	/*
6154	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
6155	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
6156	* (e.g. TPR below threshold).
6157	*/
6158	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
6159	{
6160	vmxHCClearIntWindowExitVmcs(pVCpu, pVmcsInfo);
6161	vmxHCClearNmiWindowExitVmcs(pVCpu, pVmcsInfo);
6162	}
6163
6164	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
6165	and if we're injecting an event we should have a TRPM trap pending. */
6166	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6167	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
6168	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6169	#endif
6170
6171	/* Save guest state and restore host state bits. */
6172	int rc = vmxHCLeaveSession(pVCpu);
6173	AssertRCReturn(rc, rc);
6174	STAM_COUNTER_DEC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchLongJmpToR3);
6175
6176	/* Thread-context hooks are unregistered at this point!!! */
6177	/* Ring-3 callback notifications are unregistered at this point!!! */
6178
6179	/* Sync recompiler state. */
6180	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6181	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6182	\| CPUM_CHANGED_LDTR
6183	\| CPUM_CHANGED_GDTR
6184	\| CPUM_CHANGED_IDTR
6185	\| CPUM_CHANGED_TR
6186	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6187	if ( pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging
6188	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
6189	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6190
6191	Assert(!pVCpu->hmr0.s.fClearTrapFlag);
6192
6193	/* Update the exit-to-ring 3 reason. */
6194	VCPU_2_VMXSTATE(pVCpu).rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
6195
6196	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6197	if ( rcExit != VINF_EM_RAW_INTERRUPT
6198	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
6199	{
6200	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
6201	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
6202	}
6203
6204	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchExitToR3);
6205	VMMRZCallRing3Enable(pVCpu);
6206	return rc;
6207	}
6208
6209
6210	/**
6211	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6212	* stack.
6213	*
6214	* @returns Strict VBox status code (i.e. informational status codes too).
6215	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6216	* @param pVCpu The cross context virtual CPU structure.
6217	* @param uValue The value to push to the guest stack.
6218	*/
6219	static VBOXSTRICTRC vmxHCRealModeGuestStackPush(PVMCPUCC pVCpu, uint16_t uValue)
6220	{
6221	/*
6222	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6223	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6224	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6225	*/
6226	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6227	if (pCtx->sp == 1)
6228	return VINF_EM_RESET;
6229	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6230	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
6231	AssertRC(rc);
6232	return rc;
6233	}
6234	#endif /* !IN_RING */
6235
6236	/**
6237	* Injects an event into the guest upon VM-entry by updating the relevant fields
6238	* in the VM-entry area in the VMCS.
6239	*
6240	* @returns Strict VBox status code (i.e. informational status codes too).
6241	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6242	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6243	*
6244	* @param pVCpu The cross context virtual CPU structure.
6245	* @param pVmxTransient The VMX-transient structure.
6246	* @param pEvent The event being injected.
6247	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state. This
6248	* will be updated if necessary. This cannot not be NULL.
6249	* @param fStepping Whether we're single-stepping guest execution and should
6250	* return VINF_EM_DBG_STEPPED if the event is injected
6251	* directly (registers modified by us, not by hardware on
6252	* VM-entry).
6253	*/
6254	static VBOXSTRICTRC vmxHCInjectEventVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNestedGuest, PCHMEVENT pEvent, bool fStepping,
6255	uint32_t *pfIntrState)
6256	{
6257	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6258	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
6259	Assert(pfIntrState);
6260
6261	#ifndef IN_RING0
6262	RT_NOREF(fIsNestedGuest, fStepping, pfIntrState);
6263	#endif
6264
6265	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6266	uint32_t u32IntInfo = pEvent->u64IntInfo;
6267	uint32_t const u32ErrCode = pEvent->u32ErrCode;
6268	uint32_t const cbInstr = pEvent->cbInstr;
6269	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
6270	uint8_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
6271	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
6272
6273	#ifdef VBOX_STRICT
6274	/*
6275	* Validate the error-code-valid bit for hardware exceptions.
6276	* No error codes for exceptions in real-mode.
6277	*
6278	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
6279	*/
6280	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
6281	&& !CPUMIsGuestInRealModeEx(pCtx))
6282	{
6283	switch (uVector)
6284	{
6285	case X86_XCPT_PF:
6286	case X86_XCPT_DF:
6287	case X86_XCPT_TS:
6288	case X86_XCPT_NP:
6289	case X86_XCPT_SS:
6290	case X86_XCPT_GP:
6291	case X86_XCPT_AC:
6292	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
6293	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
6294	RT_FALL_THRU();
6295	default:
6296	break;
6297	}
6298	}
6299
6300	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6301	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
6302	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
6303	#endif
6304
6305	RT_NOREF(uVector);
6306	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
6307	\|\| uIntType == VMX_EXIT_INT_INFO_TYPE_NMI
6308	\|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT
6309	\|\| uIntType == VMX_EXIT_INT_INFO_TYPE_SW_XCPT)
6310	{
6311	Assert(uVector <= X86_XCPT_LAST);
6312	Assert(uIntType != VMX_EXIT_INT_INFO_TYPE_NMI \|\| uVector == X86_XCPT_NMI);
6313	Assert(uIntType != VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT \|\| uVector == X86_XCPT_DB);
6314	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).aStatInjectedXcpts[uVector]);
6315	}
6316	else
6317	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).aStatInjectedIrqs[uVector & MASK_INJECT_IRQ_STAT]);
6318
6319	/*
6320	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
6321	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
6322	* interrupt handler in the (real-mode) guest.
6323	*
6324	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
6325	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6326	*/
6327	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
6328	{
6329	#ifdef IN_RING0
6330	if (pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fUnrestrictedGuest)
6331	#endif
6332	{
6333	/*
6334	* For CPUs with unrestricted guest execution enabled and with the guest
6335	* in real-mode, we must not set the deliver-error-code bit.
6336	*
6337	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6338	*/
6339	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
6340	}
6341	#ifdef IN_RING0
6342	else
6343	{
6344	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6345	Assert(PDMVmmDevHeapIsEnabled(pVM));
6346	Assert(pVM->hm.s.vmx.pRealModeTSS);
6347	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
6348
6349	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
6350	int rc2 = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
6351	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
6352	AssertRCReturn(rc2, rc2);
6353
6354	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6355	size_t const cbIdtEntry = sizeof(X86IDTR16);
6356	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
6357	{
6358	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6359	if (uVector == X86_XCPT_DF)
6360	return VINF_EM_RESET;
6361
6362	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
6363	No error codes for exceptions in real-mode. */
6364	if (uVector == X86_XCPT_GP)
6365	{
6366	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
6367	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
6368	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6369	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6370	HMEVENT EventXcptDf;
6371	RT_ZERO(EventXcptDf);
6372	EventXcptDf.u64IntInfo = uXcptDfInfo;
6373	return vmxHCInjectEventVmcs(pVCpu, pVmcsInfo, fIsNestedGuest, &EventXcptDf, fStepping, pfIntrState);
6374	}
6375
6376	/*
6377	* If we're injecting an event with no valid IDT entry, inject a #GP.
6378	* No error codes for exceptions in real-mode.
6379	*
6380	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
6381	*/
6382	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
6383	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
6384	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6385	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6386	HMEVENT EventXcptGp;
6387	RT_ZERO(EventXcptGp);
6388	EventXcptGp.u64IntInfo = uXcptGpInfo;
6389	return vmxHCInjectEventVmcs(pVCpu, pVmcsInfo, fIsNestedGuest, &EventXcptGp, fStepping, pfIntrState);
6390	}
6391
6392	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6393	uint16_t uGuestIp = pCtx->ip;
6394	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
6395	{
6396	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6397	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6398	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
6399	}
6400	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
6401	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
6402
6403	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6404	X86IDTR16 IdtEntry;
6405	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
6406	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
6407	AssertRCReturn(rc2, rc2);
6408
6409	/* Construct the stack frame for the interrupt/exception handler. */
6410	VBOXSTRICTRC rcStrict;
6411	rcStrict = vmxHCRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
6412	if (rcStrict == VINF_SUCCESS)
6413	{
6414	rcStrict = vmxHCRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
6415	if (rcStrict == VINF_SUCCESS)
6416	rcStrict = vmxHCRealModeGuestStackPush(pVCpu, uGuestIp);
6417	}
6418
6419	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6420	if (rcStrict == VINF_SUCCESS)
6421	{
6422	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6423	pCtx->rip = IdtEntry.offSel;
6424	pCtx->cs.Sel = IdtEntry.uSel;
6425	pCtx->cs.ValidSel = IdtEntry.uSel;
6426	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
6427	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
6428	&& uVector == X86_XCPT_PF)
6429	pCtx->cr2 = GCPtrFault;
6430
6431	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
6432	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
6433	\| HM_CHANGED_GUEST_RSP);
6434
6435	/*
6436	* If we delivered a hardware exception (other than an NMI) and if there was
6437	* block-by-STI in effect, we should clear it.
6438	*/
6439	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
6440	{
6441	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
6442	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
6443	Log4Func(("Clearing inhibition due to STI\n"));
6444	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
6445	}
6446
6447	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
6448	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
6449
6450	/*
6451	* The event has been truly dispatched to the guest. Mark it as no longer pending so
6452	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
6453	*/
6454	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false;
6455
6456	/*
6457	* If we eventually support nested-guest execution without unrestricted guest execution,
6458	* we should set fInterceptEvents here.
6459	*/
6460	Assert(!fIsNestedGuest);
6461
6462	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
6463	if (fStepping)
6464	rcStrict = VINF_EM_DBG_STEPPED;
6465	}
6466	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
6467	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6468	return rcStrict;
6469	}
6470	#else
6471	RT_NOREF(pVmcsInfo);
6472	#endif
6473	}
6474
6475	/*
6476	* Validate.
6477	*/
6478	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6479	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
6480
6481	/*
6482	* Inject the event into the VMCS.
6483	*/
6484	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
6485	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
6486	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6487	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6488	AssertRC(rc);
6489
6490	/*
6491	* Update guest CR2 if this is a page-fault.
6492	*/
6493	if (VMX_ENTRY_INT_INFO_IS_XCPT_PF(u32IntInfo))
6494	pCtx->cr2 = GCPtrFault;
6495
6496	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
6497	return VINF_SUCCESS;
6498	}
6499
6500
6501	/**
6502	* Evaluates the event to be delivered to the guest and sets it as the pending
6503	* event.
6504	*
6505	* Toggling of interrupt force-flags here is safe since we update TRPM on premature
6506	* exits to ring-3 before executing guest code, see vmxHCExitToRing3(). We must
6507	* NOT restore these force-flags.
6508	*
6509	* @returns Strict VBox status code (i.e. informational status codes too).
6510	* @param pVCpu The cross context virtual CPU structure.
6511	* @param pVmcsInfo The VMCS information structure.
6512	* @param fIsNestedGuest Flag whether the evaluation happens for a nestd guest.
6513	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
6514	*/
6515	static VBOXSTRICTRC vmxHCEvaluatePendingEvent(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNestedGuest, uint32_t *pfIntrState)
6516	{
6517	Assert(pfIntrState);
6518	Assert(!TRPMHasTrap(pVCpu));
6519
6520	/*
6521	* Compute/update guest-interruptibility state related FFs.
6522	* The FFs will be used below while evaluating events to be injected.
6523	*/
6524	*pfIntrState = vmxHCGetGuestIntrStateAndUpdateFFs(pVCpu);
6525
6526	/*
6527	* Evaluate if a new event needs to be injected.
6528	* An event that's already pending has already performed all necessary checks.
6529	*/
6530	if ( !VCPU_2_VMXSTATE(pVCpu).Event.fPending
6531	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6532	{
6533	/** @todo SMI. SMIs take priority over NMIs. */
6534
6535	/*
6536	* NMIs.
6537	* NMIs take priority over external interrupts.
6538	*/
6539	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6540	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6541	#endif
6542	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
6543	{
6544	/*
6545	* For a guest, the FF always indicates the guest's ability to receive an NMI.
6546	*
6547	* For a nested-guest, the FF always indicates the outer guest's ability to
6548	* receive an NMI while the guest-interruptibility state bit depends on whether
6549	* the nested-hypervisor is using virtual-NMIs.
6550	*/
6551	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
6552	{
6553	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6554	if ( fIsNestedGuest
6555	&& CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_NMI_EXIT))
6556	return IEMExecVmxVmexitXcptNmi(pVCpu);
6557	#endif
6558	vmxHCSetPendingXcptNmi(pVCpu);
6559	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6560	Log4Func(("NMI pending injection\n"));
6561
6562	/* We've injected the NMI, bail. */
6563	return VINF_SUCCESS;
6564	}
6565	else if (!fIsNestedGuest)
6566	vmxHCSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
6567	}
6568
6569	/*
6570	* External interrupts (PIC/APIC).
6571	* Once PDMGetInterrupt() returns a valid interrupt we -must- deliver it.
6572	* We cannot re-request the interrupt from the controller again.
6573	*/
6574	if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
6575	&& !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
6576	{
6577	Assert(!DBGFIsStepping(pVCpu));
6578	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
6579	AssertRC(rc);
6580
6581	/*
6582	* We must not check EFLAGS directly when executing a nested-guest, use
6583	* CPUMIsGuestPhysIntrEnabled() instead as EFLAGS.IF does not control the blocking of
6584	* external interrupts when "External interrupt exiting" is set. This fixes a nasty
6585	* SMP hang while executing nested-guest VCPUs on spinlocks which aren't rescued by
6586	* other VM-exits (like a preemption timer), see @bugref{9562#c18}.
6587	*
6588	* See Intel spec. 25.4.1 "Event Blocking".
6589	*/
6590	if (CPUMIsGuestPhysIntrEnabled(pVCpu))
6591	{
6592	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6593	if ( fIsNestedGuest
6594	&& CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
6595	{
6596	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0 /* uVector /, true / fIntPending */);
6597	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
6598	return rcStrict;
6599	}
6600	#endif
6601	uint8_t u8Interrupt;
6602	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6603	if (RT_SUCCESS(rc))
6604	{
6605	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6606	if ( fIsNestedGuest
6607	&& CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
6608	{
6609	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
6610	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
6611	return rcStrict;
6612	}
6613	#endif
6614	vmxHCSetPendingExtInt(pVCpu, u8Interrupt);
6615	Log4Func(("External interrupt (%#x) pending injection\n", u8Interrupt));
6616	}
6617	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
6618	{
6619	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchTprMaskedIrq);
6620
6621	if ( !fIsNestedGuest
6622	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
6623	vmxHCApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
6624	/* else: for nested-guests, TPR threshold is picked up while merging VMCS controls. */
6625
6626	/*
6627	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
6628	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
6629	* need to re-set this force-flag here.
6630	*/
6631	}
6632	else
6633	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchGuestIrq);
6634
6635	/* We've injected the interrupt or taken necessary action, bail. */
6636	return VINF_SUCCESS;
6637	}
6638	if (!fIsNestedGuest)
6639	vmxHCSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
6640	}
6641	}
6642	else if (!fIsNestedGuest)
6643	{
6644	/*
6645	* An event is being injected or we are in an interrupt shadow. Check if another event is
6646	* pending. If so, instruct VT-x to cause a VM-exit as soon as the guest is ready to accept
6647	* the pending event.
6648	*/
6649	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
6650	vmxHCSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
6651	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
6652	&& !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
6653	vmxHCSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
6654	}
6655	/* else: for nested-guests, NMI/interrupt-window exiting will be picked up when merging VMCS controls. */
6656
6657	return VINF_SUCCESS;
6658	}
6659
6660
6661	/**
6662	* Injects any pending events into the guest if the guest is in a state to
6663	* receive them.
6664	*
6665	* @returns Strict VBox status code (i.e. informational status codes too).
6666	* @param pVCpu The cross context virtual CPU structure.
6667	* @param fIsNestedGuest Flag whether the event injection happens for a nested guest.
6668	* @param fIntrState The VT-x guest-interruptibility state.
6669	* @param fStepping Whether we are single-stepping the guest using the
6670	* hypervisor debugger and should return
6671	* VINF_EM_DBG_STEPPED if the event was dispatched
6672	* directly.
6673	*/
6674	static VBOXSTRICTRC vmxHCInjectPendingEvent(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNestedGuest, uint32_t fIntrState, bool fStepping)
6675	{
6676	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6677	#ifdef IN_RING0
6678	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6679	#endif
6680
6681	#ifdef VBOX_STRICT
6682	/*
6683	* Verify guest-interruptibility state.
6684	*
6685	* We put this in a scoped block so we do not accidentally use fBlockSti or fBlockMovSS,
6686	* since injecting an event may modify the interruptibility state and we must thus always
6687	* use fIntrState.
6688	*/
6689	{
6690	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
6691	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
6692	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
6693	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6694	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
6695	Assert(!TRPMHasTrap(pVCpu));
6696	NOREF(fBlockMovSS); NOREF(fBlockSti);
6697	}
6698	#endif
6699
6700	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
6701	if (VCPU_2_VMXSTATE(pVCpu).Event.fPending)
6702	{
6703	/*
6704	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
6705	* pending even while injecting an event and in this case, we want a VM-exit as soon as
6706	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
6707	*
6708	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
6709	*/
6710	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo);
6711	#ifdef VBOX_STRICT
6712	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6713	{
6714	Assert(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
6715	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI));
6716	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
6717	}
6718	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
6719	{
6720	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI));
6721	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI));
6722	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
6723	}
6724	#endif
6725	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
6726	uIntType));
6727
6728	/*
6729	* Inject the event and get any changes to the guest-interruptibility state.
6730	*
6731	* The guest-interruptibility state may need to be updated if we inject the event
6732	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
6733	*/
6734	rcStrict = vmxHCInjectEventVmcs(pVCpu, pVmcsInfo, fIsNestedGuest, &VCPU_2_VMXSTATE(pVCpu).Event, fStepping, &fIntrState);
6735	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
6736
6737	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6738	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectInterrupt);
6739	else
6740	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectXcpt);
6741	}
6742
6743	/*
6744	* Deliver any pending debug exceptions if the guest is single-stepping using EFLAGS.TF and
6745	* is an interrupt shadow (block-by-STI or block-by-MOV SS).
6746	*/
6747	if ( (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
6748	&& !fIsNestedGuest)
6749	{
6750	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
6751
6752	if (!VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
6753	{
6754	/*
6755	* Set or clear the BS bit depending on whether the trap flag is active or not. We need
6756	* to do both since we clear the BS bit from the VMCS while exiting to ring-3.
6757	*/
6758	Assert(!DBGFIsStepping(pVCpu));
6759	uint8_t const fTrapFlag = !!(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_TF);
6760	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, fTrapFlag << VMX_BF_VMCS_PENDING_DBG_XCPT_BS_SHIFT);
6761	AssertRC(rc);
6762	}
6763	else
6764	{
6765	/*
6766	* We must not deliver a debug exception when single-stepping over STI/Mov-SS in the
6767	* hypervisor debugger using EFLAGS.TF but rather clear interrupt inhibition. However,
6768	* we take care of this case in vmxHCExportSharedDebugState and also the case if
6769	* we use MTF, so just make sure it's called before executing guest-code.
6770	*/
6771	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_DR_MASK);
6772	}
6773	}
6774	/* else: for nested-guest currently handling while merging controls. */
6775
6776	/*
6777	* Finally, update the guest-interruptibility state.
6778	*
6779	* This is required for the real-on-v86 software interrupt injection, for
6780	* pending debug exceptions as well as updates to the guest state from ring-3 (IEM).
6781	*/
6782	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, fIntrState);
6783	AssertRC(rc);
6784
6785	/*
6786	* There's no need to clear the VM-entry interruption-information field here if we're not
6787	* injecting anything. VT-x clears the valid bit on every VM-exit.
6788	*
6789	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6790	*/
6791
6792	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
6793	return rcStrict;
6794	}
6795
6796
6797	/**
6798	* Exports the guest state into the VMCS guest-state area.
6799	*
6800	* The will typically be done before VM-entry when the guest-CPU state and the
6801	* VMCS state may potentially be out of sync.
6802	*
6803	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
6804	* VM-entry controls.
6805	* Sets up the appropriate VMX non-root function to execute guest code based on
6806	* the guest CPU mode.
6807	*
6808	* @returns VBox strict status code.
6809	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
6810	* without unrestricted guest execution and the VMMDev is not presently
6811	* mapped (e.g. EFI32).
6812	*
6813	* @param pVCpu The cross context virtual CPU structure.
6814	* @param pVmxTransient The VMX-transient structure.
6815	*
6816	* @remarks No-long-jump zone!!!
6817	*/
6818	static VBOXSTRICTRC vmxHCExportGuestState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6819	{
6820	AssertPtr(pVCpu);
6821	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6822	LogFlowFunc(("pVCpu=%p\n", pVCpu));
6823
6824	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExportGuestState, x);
6825
6826	#ifdef IN_RING0
6827	/*
6828	* Determine real-on-v86 mode.
6829	* Used when the guest is in real-mode and unrestricted guest execution is not used.
6830	*/
6831	PVMXVMCSINFOSHARED pVmcsInfoShared = pVmxTransient->pVmcsInfo->pShared;
6832	if ( pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fUnrestrictedGuest
6833	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
6834	pVmcsInfoShared->RealMode.fRealOnV86Active = false;
6835	else
6836	{
6837	Assert(!pVmxTransient->fIsNestedGuest);
6838	pVmcsInfoShared->RealMode.fRealOnV86Active = true;
6839	}
6840	#endif
6841
6842	/*
6843	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
6844	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
6845	*/
6846	int rc = vmxHCExportGuestEntryExitCtls(pVCpu, pVmxTransient);
6847	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
6848
6849	rc = vmxHCExportGuestCR0(pVCpu, pVmxTransient);
6850	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
6851
6852	VBOXSTRICTRC rcStrict = vmxHCExportGuestCR3AndCR4(pVCpu, pVmxTransient);
6853	if (rcStrict == VINF_SUCCESS)
6854	{ /* likely */ }
6855	else
6856	{
6857	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
6858	return rcStrict;
6859	}
6860
6861	rc = vmxHCExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
6862	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
6863
6864	rc = vmxHCExportGuestMsrs(pVCpu, pVmxTransient);
6865	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
6866
6867	vmxHCExportGuestApicTpr(pVCpu, pVmxTransient);
6868	vmxHCExportGuestXcptIntercepts(pVCpu, pVmxTransient);
6869	vmxHCExportGuestRip(pVCpu);
6870	vmxHCExportGuestRsp(pVCpu);
6871	vmxHCExportGuestRflags(pVCpu, pVmxTransient);
6872
6873	rc = vmxHCExportGuestHwvirtState(pVCpu, pVmxTransient);
6874	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
6875
6876	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
6877	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
6878	\| HM_CHANGED_GUEST_CR2
6879	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
6880	\| HM_CHANGED_GUEST_X87
6881	\| HM_CHANGED_GUEST_SSE_AVX
6882	\| HM_CHANGED_GUEST_OTHER_XSAVE
6883	\| HM_CHANGED_GUEST_XCRx
6884	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
6885	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
6886	\| HM_CHANGED_GUEST_TSC_AUX
6887	\| HM_CHANGED_GUEST_OTHER_MSRS
6888	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
6889
6890	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExportGuestState, x);
6891	return rc;
6892	}
6893
6894
6895	/**
6896	* Exports the state shared between the host and guest into the VMCS.
6897	*
6898	* @param pVCpu The cross context virtual CPU structure.
6899	* @param pVmxTransient The VMX-transient structure.
6900	*
6901	* @remarks No-long-jump zone!!!
6902	*/
6903	static void vmxHCExportSharedState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6904	{
6905	#ifdef IN_RING0
6906	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6907	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6908	#endif
6909
6910	if (VCPU_2_VMXSTATE(pVCpu).fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
6911	{
6912	int rc = vmxHCExportSharedDebugState(pVCpu, pVmxTransient);
6913	AssertRC(rc);
6914	VCPU_2_VMXSTATE(pVCpu).fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
6915
6916	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
6917	if (VCPU_2_VMXSTATE(pVCpu).fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
6918	vmxHCExportGuestRflags(pVCpu, pVmxTransient);
6919	}
6920
6921	#ifdef IN_RING0
6922	if (VCPU_2_VMXSTATE(pVCpu).fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
6923	{
6924	vmxHCLazyLoadGuestMsrs(pVCpu);
6925	VCPU_2_VMXSTATE(pVCpu).fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
6926	}
6927	#endif
6928
6929	AssertMsg(!(VCPU_2_VMXSTATE(pVCpu).fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
6930	("fCtxChanged=%#RX64\n", VCPU_2_VMXSTATE(pVCpu).fCtxChanged));
6931	}
6932
6933
6934	/**
6935	* Worker for loading the guest-state bits in the inner VT-x execution loop.
6936	*
6937	* @returns Strict VBox status code (i.e. informational status codes too).
6938	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
6939	* without unrestricted guest execution and the VMMDev is not presently
6940	* mapped (e.g. EFI32).
6941	*
6942	* @param pVCpu The cross context virtual CPU structure.
6943	* @param pVmxTransient The VMX-transient structure.
6944	*
6945	* @remarks No-long-jump zone!!!
6946	*/
6947	static VBOXSTRICTRC vmxHCExportGuestStateOptimal(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6948	{
6949	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6950	#ifdef IN_RING0
6951	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6952	#endif
6953
6954	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
6955	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
6956	#endif
6957
6958	/*
6959	* For many VM-exits only RIP/RSP/RFLAGS (and HWVIRT state when executing a nested-guest)
6960	* changes. First try to export only these without going through all other changed-flag checks.
6961	*/
6962	VBOXSTRICTRC rcStrict;
6963	uint64_t const fCtxMask = HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE;
6964	uint64_t const fMinimalMask = HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT;
6965	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged);
6966
6967	/* If only RIP/RSP/RFLAGS/HWVIRT changed, export only those (quicker, happens more often).*/
6968	if ( (fCtxChanged & fMinimalMask)
6969	&& !(fCtxChanged & (fCtxMask & ~fMinimalMask)))
6970	{
6971	vmxHCExportGuestRip(pVCpu);
6972	vmxHCExportGuestRsp(pVCpu);
6973	vmxHCExportGuestRflags(pVCpu, pVmxTransient);
6974	rcStrict = vmxHCExportGuestHwvirtState(pVCpu, pVmxTransient);
6975	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExportMinimal);
6976	}
6977	/* If anything else also changed, go through the full export routine and export as required. */
6978	else if (fCtxChanged & fCtxMask)
6979	{
6980	rcStrict = vmxHCExportGuestState(pVCpu, pVmxTransient);
6981	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
6982	{ /* likely */}
6983	else
6984	{
6985	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
6986	VBOXSTRICTRC_VAL(rcStrict)));
6987	#ifdef IN_RING0
6988	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6989	#endif
6990	return rcStrict;
6991	}
6992	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExportFull);
6993	}
6994	/* Nothing changed, nothing to load here. */
6995	else
6996	rcStrict = VINF_SUCCESS;
6997
6998	#ifdef VBOX_STRICT
6999	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7000	uint64_t const fCtxChangedCur = ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged);
7001	AssertMsg(!(fCtxChangedCur & fCtxMask), ("fCtxChangedCur=%#RX64\n", fCtxChangedCur));
7002	#endif
7003	return rcStrict;
7004	}
7005
7006
7007	/**
7008	* Tries to determine what part of the guest-state VT-x has deemed as invalid
7009	* and update error record fields accordingly.
7010	*
7011	* @returns VMX_IGS_* error codes.
7012	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
7013	* wrong with the guest state.
7014	*
7015	* @param pVCpu The cross context virtual CPU structure.
7016	* @param pVmcsInfo The VMCS info. object.
7017	*
7018	* @remarks This function assumes our cache of the VMCS controls
7019	* are valid, i.e. vmxHCCheckCachedVmcsCtls() succeeded.
7020	*/
7021	static uint32_t vmxHCCheckGuestState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
7022	{
7023	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
7024	#define HMVMX_CHECK_BREAK(expr, err) do { \
7025	if (!(expr)) { uError = (err); break; } \
7026	} while (0)
7027
7028	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7029	uint32_t uError = VMX_IGS_ERROR;
7030	uint32_t u32IntrState = 0;
7031	#ifdef IN_RING0
7032	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7033	bool const fUnrestrictedGuest = VM_IS_VMX_UNRESTRICTED_GUEST(pVM);
7034	#else
7035	bool const fUnrestrictedGuest = true;
7036	#endif
7037	do
7038	{
7039	int rc;
7040
7041	/*
7042	* Guest-interruptibility state.
7043	*
7044	* Read this first so that any check that fails prior to those that actually
7045	* require the guest-interruptibility state would still reflect the correct
7046	* VMCS value and avoids causing further confusion.
7047	*/
7048	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
7049	AssertRC(rc);
7050
7051	uint32_t u32Val;
7052	uint64_t u64Val;
7053
7054	/*
7055	* CR0.
7056	*/
7057	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
7058	uint64_t fSetCr0 = (g_HmMsrs.u.vmx.u64Cr0Fixed0 & g_HmMsrs.u.vmx.u64Cr0Fixed1);
7059	uint64_t const fZapCr0 = (g_HmMsrs.u.vmx.u64Cr0Fixed0 \| g_HmMsrs.u.vmx.u64Cr0Fixed1);
7060	/* Exceptions for unrestricted guest execution for CR0 fixed bits (PE, PG).
7061	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
7062	if (fUnrestrictedGuest)
7063	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
7064
7065	uint64_t u64GuestCr0;
7066	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0, &u64GuestCr0);
7067	AssertRC(rc);
7068	HMVMX_CHECK_BREAK((u64GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
7069	HMVMX_CHECK_BREAK(!(u64GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
7070	if ( !fUnrestrictedGuest
7071	&& (u64GuestCr0 & X86_CR0_PG)
7072	&& !(u64GuestCr0 & X86_CR0_PE))
7073	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
7074
7075	/*
7076	* CR4.
7077	*/
7078	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
7079	uint64_t const fSetCr4 = (g_HmMsrs.u.vmx.u64Cr4Fixed0 & g_HmMsrs.u.vmx.u64Cr4Fixed1);
7080	uint64_t const fZapCr4 = (g_HmMsrs.u.vmx.u64Cr4Fixed0 \| g_HmMsrs.u.vmx.u64Cr4Fixed1);
7081
7082	uint64_t u64GuestCr4;
7083	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR4, &u64GuestCr4);
7084	AssertRC(rc);
7085	HMVMX_CHECK_BREAK((u64GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
7086	HMVMX_CHECK_BREAK(!(u64GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
7087
7088	/*
7089	* IA32_DEBUGCTL MSR.
7090	*/
7091	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
7092	AssertRC(rc);
7093	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
7094	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
7095	{
7096	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
7097	}
7098	uint64_t u64DebugCtlMsr = u64Val;
7099
7100	#ifdef VBOX_STRICT
7101	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY, &u32Val);
7102	AssertRC(rc);
7103	Assert(u32Val == pVmcsInfo->u32EntryCtls);
7104	#endif
7105	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
7106
7107	/*
7108	* RIP and RFLAGS.
7109	*/
7110	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RIP, &u64Val);
7111	AssertRC(rc);
7112	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
7113	if ( !fLongModeGuest
7114	\|\| !pCtx->cs.Attr.n.u1Long)
7115	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
7116	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
7117	* must be identical if the "IA-32e mode guest" VM-entry
7118	* control is 1 and CS.L is 1. No check applies if the
7119	* CPU supports 64 linear-address bits. */
7120
7121	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
7122	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RFLAGS, &u64Val);
7123	AssertRC(rc);
7124	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
7125	VMX_IGS_RFLAGS_RESERVED);
7126	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
7127	uint32_t const u32Eflags = u64Val;
7128
7129	if ( fLongModeGuest
7130	\|\| ( fUnrestrictedGuest
7131	&& !(u64GuestCr0 & X86_CR0_PE)))
7132	{
7133	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
7134	}
7135
7136	uint32_t u32EntryInfo;
7137	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7138	AssertRC(rc);
7139	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
7140	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
7141
7142	/*
7143	* 64-bit checks.
7144	*/
7145	if (fLongModeGuest)
7146	{
7147	HMVMX_CHECK_BREAK(u64GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
7148	HMVMX_CHECK_BREAK(u64GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
7149	}
7150
7151	if ( !fLongModeGuest
7152	&& (u64GuestCr4 & X86_CR4_PCIDE))
7153	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
7154
7155	/** @todo CR3 field must be such that bits 63:52 and bits in the range
7156	* 51:32 beyond the processor's physical-address width are 0. */
7157
7158	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
7159	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
7160	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
7161
7162	#ifdef IN_RING0
7163	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
7164	AssertRC(rc);
7165	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
7166
7167	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
7168	AssertRC(rc);
7169	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
7170	#endif
7171
7172	/*
7173	* PERF_GLOBAL MSR.
7174	*/
7175	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
7176	{
7177	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
7178	AssertRC(rc);
7179	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
7180	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
7181	}
7182
7183	/*
7184	* PAT MSR.
7185	*/
7186	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
7187	{
7188	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
7189	AssertRC(rc);
7190	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
7191	for (unsigned i = 0; i < 8; i++)
7192	{
7193	uint8_t u8Val = (u64Val & 0xff);
7194	if ( u8Val != 0 /* UC */
7195	&& u8Val != 1 /* WC */
7196	&& u8Val != 4 /* WT */
7197	&& u8Val != 5 /* WP */
7198	&& u8Val != 6 /* WB */
7199	&& u8Val != 7 /* UC- */)
7200	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
7201	u64Val >>= 8;
7202	}
7203	}
7204
7205	/*
7206	* EFER MSR.
7207	*/
7208	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
7209	{
7210	Assert(g_fHmVmxSupportsVmcsEfer);
7211	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
7212	AssertRC(rc);
7213	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
7214	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
7215	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
7216	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
7217	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
7218	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
7219	* iemVmxVmentryCheckGuestState(). */
7220	HMVMX_CHECK_BREAK( fUnrestrictedGuest
7221	\|\| !(u64GuestCr0 & X86_CR0_PG)
7222	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
7223	VMX_IGS_EFER_LMA_LME_MISMATCH);
7224	}
7225
7226	/*
7227	* Segment registers.
7228	*/
7229	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
7230	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
7231	if (!(u32Eflags & X86_EFL_VM))
7232	{
7233	/* CS */
7234	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
7235	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
7236	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
7237	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
7238	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
7239	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
7240	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
7241	/* CS cannot be loaded with NULL in protected mode. */
7242	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
7243	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
7244	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
7245	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
7246	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
7247	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
7248	else if (fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
7249	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
7250	else
7251	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
7252
7253	/* SS */
7254	HMVMX_CHECK_BREAK( fUnrestrictedGuest
7255	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
7256	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
7257	if ( !(pCtx->cr0 & X86_CR0_PE)
7258	\|\| pCtx->cs.Attr.n.u4Type == 3)
7259	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
7260
7261	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
7262	{
7263	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
7264	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
7265	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
7266	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
7267	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
7268	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
7269	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
7270	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
7271	}
7272
7273	/* DS, ES, FS, GS - only check for usable selectors, see vmxHCExportGuestSReg(). */
7274	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
7275	{
7276	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
7277	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
7278	HMVMX_CHECK_BREAK( fUnrestrictedGuest
7279	\|\| pCtx->ds.Attr.n.u4Type > 11
7280	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
7281	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
7282	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
7283	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
7284	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
7285	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
7286	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
7287	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
7288	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
7289	}
7290	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
7291	{
7292	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
7293	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
7294	HMVMX_CHECK_BREAK( fUnrestrictedGuest
7295	\|\| pCtx->es.Attr.n.u4Type > 11
7296	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
7297	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
7298	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
7299	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
7300	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
7301	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
7302	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
7303	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
7304	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
7305	}
7306	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
7307	{
7308	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
7309	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
7310	HMVMX_CHECK_BREAK( fUnrestrictedGuest
7311	\|\| pCtx->fs.Attr.n.u4Type > 11
7312	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
7313	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
7314	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
7315	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
7316	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
7317	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
7318	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
7319	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
7320	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
7321	}
7322	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
7323	{
7324	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
7325	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
7326	HMVMX_CHECK_BREAK( fUnrestrictedGuest
7327	\|\| pCtx->gs.Attr.n.u4Type > 11
7328	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
7329	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
7330	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
7331	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
7332	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
7333	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
7334	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
7335	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
7336	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
7337	}
7338	/* 64-bit capable CPUs. */
7339	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
7340	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
7341	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
7342	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
7343	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
7344	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
7345	VMX_IGS_LONGMODE_SS_BASE_INVALID);
7346	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
7347	VMX_IGS_LONGMODE_DS_BASE_INVALID);
7348	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
7349	VMX_IGS_LONGMODE_ES_BASE_INVALID);
7350	}
7351	else
7352	{
7353	/* V86 mode checks. */
7354	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
7355	if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
7356	{
7357	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
7358	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
7359	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
7360	}
7361	else
7362	{
7363	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
7364	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
7365	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
7366	}
7367
7368	/* CS */
7369	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
7370	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
7371	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
7372	/* SS */
7373	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
7374	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
7375	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
7376	/* DS */
7377	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
7378	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
7379	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
7380	/* ES */
7381	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
7382	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
7383	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
7384	/* FS */
7385	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
7386	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
7387	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
7388	/* GS */
7389	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
7390	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
7391	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
7392	/* 64-bit capable CPUs. */
7393	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
7394	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
7395	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
7396	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
7397	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
7398	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
7399	VMX_IGS_LONGMODE_SS_BASE_INVALID);
7400	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
7401	VMX_IGS_LONGMODE_DS_BASE_INVALID);
7402	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
7403	VMX_IGS_LONGMODE_ES_BASE_INVALID);
7404	}
7405
7406	/*
7407	* TR.
7408	*/
7409	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
7410	/* 64-bit capable CPUs. */
7411	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
7412	if (fLongModeGuest)
7413	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
7414	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
7415	else
7416	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
7417	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
7418	VMX_IGS_TR_ATTR_TYPE_INVALID);
7419	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
7420	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
7421	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
7422	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
7423	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
7424	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
7425	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
7426	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
7427
7428	/*
7429	* GDTR and IDTR (64-bit capable checks).
7430	*/
7431	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
7432	AssertRC(rc);
7433	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
7434
7435	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
7436	AssertRC(rc);
7437	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
7438
7439	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
7440	AssertRC(rc);
7441	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
7442
7443	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
7444	AssertRC(rc);
7445	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
7446
7447	/*
7448	* Guest Non-Register State.
7449	*/
7450	/* Activity State. */
7451	uint32_t u32ActivityState;
7452	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
7453	AssertRC(rc);
7454	HMVMX_CHECK_BREAK( !u32ActivityState
7455	\|\| (u32ActivityState & RT_BF_GET(g_HmMsrs.u.vmx.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
7456	VMX_IGS_ACTIVITY_STATE_INVALID);
7457	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
7458	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
7459
7460	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
7461	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
7462	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
7463
7464	/** @todo Activity state and injecting interrupts. Left as a todo since we
7465	* currently don't use activity states but ACTIVE. */
7466
7467	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
7468	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
7469
7470	/* Guest interruptibility-state. */
7471	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
7472	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
7473	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
7474	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
7475	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
7476	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
7477	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
7478	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
7479	{
7480	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
7481	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
7482	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
7483	}
7484	else if (VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
7485	{
7486	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
7487	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
7488	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
7489	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
7490	}
7491	/** @todo Assumes the processor is not in SMM. */
7492	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
7493	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
7494	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
7495	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
7496	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
7497	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
7498	&& VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
7499	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI), VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
7500
7501	/* Pending debug exceptions. */
7502	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
7503	AssertRC(rc);
7504	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
7505	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
7506	u32Val = u64Val; /* For pending debug exceptions checks below. */
7507
7508	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
7509	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
7510	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
7511	{
7512	if ( (u32Eflags & X86_EFL_TF)
7513	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
7514	{
7515	/* Bit 14 is PendingDebug.BS. */
7516	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
7517	}
7518	if ( !(u32Eflags & X86_EFL_TF)
7519	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
7520	{
7521	/* Bit 14 is PendingDebug.BS. */
7522	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
7523	}
7524	}
7525
7526	#ifdef IN_RING0
7527	/* VMCS link pointer. */
7528	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
7529	AssertRC(rc);
7530	if (u64Val != UINT64_C(0xffffffffffffffff))
7531	{
7532	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
7533	/** @todo Bits beyond the processor's physical-address width MBZ. */
7534	/** @todo SMM checks. */
7535	Assert(pVmcsInfo->HCPhysShadowVmcs == u64Val);
7536	Assert(pVmcsInfo->pvShadowVmcs);
7537	VMXVMCSREVID VmcsRevId;
7538	VmcsRevId.u = (uint32_t )pVmcsInfo->pvShadowVmcs;
7539	HMVMX_CHECK_BREAK(VmcsRevId.n.u31RevisionId == RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_ID),
7540	VMX_IGS_VMCS_LINK_PTR_SHADOW_VMCS_ID_INVALID);
7541	HMVMX_CHECK_BREAK(VmcsRevId.n.fIsShadowVmcs == (uint32_t)!!(pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING),
7542	VMX_IGS_VMCS_LINK_PTR_NOT_SHADOW);
7543	}
7544
7545	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
7546	* not using nested paging? */
7547	if ( VM_IS_VMX_NESTED_PAGING(pVM)
7548	&& !fLongModeGuest
7549	&& CPUMIsGuestInPAEModeEx(pCtx))
7550	{
7551	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
7552	AssertRC(rc);
7553	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
7554
7555	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
7556	AssertRC(rc);
7557	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
7558
7559	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
7560	AssertRC(rc);
7561	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
7562
7563	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
7564	AssertRC(rc);
7565	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
7566	}
7567	#endif
7568
7569	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
7570	if (uError == VMX_IGS_ERROR)
7571	uError = VMX_IGS_REASON_NOT_FOUND;
7572	} while (0);
7573
7574	VCPU_2_VMXSTATE(pVCpu).u32HMError = uError;
7575	VCPU_2_VMXSTATE(pVCpu).vmx.LastError.u32GuestIntrState = u32IntrState;
7576	return uError;
7577
7578	#undef HMVMX_ERROR_BREAK
7579	#undef HMVMX_CHECK_BREAK
7580	}
7581
7582
7583	#ifdef IN_RING0
7584	/**
7585	* Map the APIC-access page for virtualizing APIC accesses.
7586	*
7587	* This can cause a longjumps to R3 due to the acquisition of the PGM lock. Hence,
7588	* this not done as part of exporting guest state, see @bugref{8721}.
7589	*
7590	* @returns VBox status code.
7591	* @param pVCpu The cross context virtual CPU structure.
7592	*/
7593	static int vmxHCMapHCApicAccessPage(PVMCPUCC pVCpu)
7594	{
7595	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7596	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
7597
7598	Assert(PDMHasApic(pVM));
7599	Assert(u64MsrApicBase);
7600
7601	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
7602	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
7603
7604	/* Unalias the existing mapping. */
7605	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7606	AssertRCReturn(rc, rc);
7607
7608	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
7609	Assert(pVM->hmr0.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
7610	rc = IOMR0MmioMapMmioHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hmr0.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
7611	AssertRCReturn(rc, rc);
7612
7613	/* Update the per-VCPU cache of the APIC base MSR. */
7614	VCPU_2_VMXSTATE(pVCpu).vmx.u64GstMsrApicBase = u64MsrApicBase;
7615	return VINF_SUCCESS;
7616	}
7617
7618
7619	/**
7620	* Worker function passed to RTMpOnSpecific() that is to be called on the target
7621	* CPU.
7622	*
7623	* @param idCpu The ID for the CPU the function is called on.
7624	* @param pvUser1 Null, not used.
7625	* @param pvUser2 Null, not used.
7626	*/
7627	static DECLCALLBACK(void) hmR0DispatchHostNmi(RTCPUID idCpu, void pvUser1, void pvUser2)
7628	{
7629	RT_NOREF3(idCpu, pvUser1, pvUser2);
7630	VMXDispatchHostNmi();
7631	}
7632
7633
7634	/**
7635	* Dispatching an NMI on the host CPU that received it.
7636	*
7637	* @returns VBox status code.
7638	* @param pVCpu The cross context virtual CPU structure.
7639	* @param pVmcsInfo The VMCS info. object corresponding to the VMCS that was
7640	* executing when receiving the host NMI in VMX non-root
7641	* operation.
7642	*/
7643	static int vmxHCExitHostNmi(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
7644	{
7645	RTCPUID const idCpu = pVmcsInfo->idHostCpuExec;
7646	Assert(idCpu != NIL_RTCPUID);
7647
7648	/*
7649	* We don't want to delay dispatching the NMI any more than we have to. However,
7650	* we have already chosen -not- to dispatch NMIs when interrupts were still disabled
7651	* after executing guest or nested-guest code for the following reasons:
7652	*
7653	* - We would need to perform VMREADs with interrupts disabled and is orders of
7654	* magnitude worse when we run as a nested hypervisor without VMCS shadowing
7655	* supported by the host hypervisor.
7656	*
7657	* - It affects the common VM-exit scenario and keeps interrupts disabled for a
7658	* longer period of time just for handling an edge case like host NMIs which do
7659	* not occur nearly as frequently as other VM-exits.
7660	*
7661	* Let's cover the most likely scenario first. Check if we are on the target CPU
7662	* and dispatch the NMI right away. This should be much faster than calling into
7663	* RTMpOnSpecific() machinery.
7664	*/
7665	bool fDispatched = false;
7666	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7667	if (idCpu == RTMpCpuId())
7668	{
7669	VMXDispatchHostNmi();
7670	fDispatched = true;
7671	}
7672	ASMSetFlags(fEFlags);
7673	if (fDispatched)
7674	{
7675	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitHostNmiInGC);
7676	return VINF_SUCCESS;
7677	}
7678
7679	/*
7680	* RTMpOnSpecific() waits until the worker function has run on the target CPU. So
7681	* there should be no race or recursion even if we are unlucky enough to be preempted
7682	* (to the target CPU) without dispatching the host NMI above.
7683	*/
7684	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitHostNmiInGCIpi);
7685	return RTMpOnSpecific(idCpu, &hmR0DispatchHostNmi, NULL /* pvUser1 /, NULL / pvUser2 */);
7686	}
7687
7688
7689	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7690	/**
7691	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
7692	* nested-guest using hardware-assisted VMX.
7693	*
7694	* @param pVCpu The cross context virtual CPU structure.
7695	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
7696	* @param pVmcsInfoGst The guest VMCS info. object.
7697	*/
7698	static void vmxHCMergeMsrBitmapNested(PCVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
7699	{
7700	uint32_t const cbMsrBitmap = X86_PAGE_4K_SIZE;
7701	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
7702	Assert(pu64MsrBitmap);
7703
7704	/*
7705	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
7706	* MSR that is intercepted by the guest is also intercepted while executing the
7707	* nested-guest using hardware-assisted VMX.
7708	*
7709	* Note! If the nested-guest is not using an MSR bitmap, every MSR must cause a
7710	* nested-guest VM-exit even if the outer guest is not intercepting some
7711	* MSRs. We cannot assume the caller has initialized the nested-guest
7712	* MSR bitmap in this case.
7713	*
7714	* The nested hypervisor may also switch whether it uses MSR bitmaps for
7715	* each of its VM-entry, hence initializing it once per-VM while setting
7716	* up the nested-guest VMCS is not sufficient.
7717	*/
7718	PCVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
7719	if (pVmcsNstGst->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
7720	{
7721	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )&pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap[0];
7722	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
7723	Assert(pu64MsrBitmapNstGst);
7724	Assert(pu64MsrBitmapGst);
7725
7726	/** @todo Detect and use EVEX.POR? */
7727	uint32_t const cFrags = cbMsrBitmap / sizeof(uint64_t);
7728	for (uint32_t i = 0; i < cFrags; i++)
7729	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
7730	}
7731	else
7732	ASMMemFill32(pu64MsrBitmap, cbMsrBitmap, UINT32_C(0xffffffff));
7733	}
7734
7735
7736	/**
7737	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
7738	* hardware-assisted VMX execution of the nested-guest.
7739	*
7740	* For a guest, we don't modify these controls once we set up the VMCS and hence
7741	* this function is never called.
7742	*
7743	* For nested-guests since the nested hypervisor provides these controls on every
7744	* nested-guest VM-entry and could potentially change them everytime we need to
7745	* merge them before every nested-guest VM-entry.
7746	*
7747	* @returns VBox status code.
7748	* @param pVCpu The cross context virtual CPU structure.
7749	*/
7750	static int vmxHCMergeVmcsNested(PVMCPUCC pVCpu)
7751	{
7752	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
7753	PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo;
7754	PCVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
7755
7756	/*
7757	* Merge the controls with the requirements of the guest VMCS.
7758	*
7759	* We do not need to validate the nested-guest VMX features specified in the nested-guest
7760	* VMCS with the features supported by the physical CPU as it's already done by the
7761	* VMLAUNCH/VMRESUME instruction emulation.
7762	*
7763	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the guest are
7764	* derived from the VMX features supported by the physical CPU.
7765	*/
7766
7767	/* Pin-based VM-execution controls. */
7768	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
7769
7770	/* Processor-based VM-execution controls. */
7771	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
7772	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
7773	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
7774	\| VMX_PROC_CTLS_MOV_DR_EXIT
7775	\| VMX_PROC_CTLS_USE_TPR_SHADOW
7776	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
7777
7778	/* Secondary processor-based VM-execution controls. */
7779	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
7780	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
7781	\| VMX_PROC_CTLS2_INVPCID
7782	\| VMX_PROC_CTLS2_VMCS_SHADOWING
7783	\| VMX_PROC_CTLS2_RDTSCP
7784	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
7785	\| VMX_PROC_CTLS2_APIC_REG_VIRT
7786	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
7787	\| VMX_PROC_CTLS2_VMFUNC));
7788
7789	/*
7790	* VM-entry controls:
7791	* These controls contains state that depends on the nested-guest state (primarily
7792	* EFER MSR) and is thus not constant between VMLAUNCH/VMRESUME and the nested-guest
7793	* VM-exit. Although the nested hypervisor cannot change it, we need to in order to
7794	* properly continue executing the nested-guest if the EFER MSR changes but does not
7795	* cause a nested-guest VM-exits.
7796	*
7797	* VM-exit controls:
7798	* These controls specify the host state on return. We cannot use the controls from
7799	* the nested hypervisor state as is as it would contain the guest state rather than
7800	* the host state. Since the host state is subject to change (e.g. preemption, trips
7801	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
7802	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
7803	*
7804	* VM-entry MSR-load:
7805	* The guest MSRs from the VM-entry MSR-load area are already loaded into the guest-CPU
7806	* context by the VMLAUNCH/VMRESUME instruction emulation.
7807	*
7808	* VM-exit MSR-store:
7809	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU context
7810	* back into the VM-exit MSR-store area.
7811	*
7812	* VM-exit MSR-load areas:
7813	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence, we
7814	* can entirely ignore what the nested hypervisor wants to load here.
7815	*/
7816
7817	/*
7818	* Exception bitmap.
7819	*
7820	* We could remove #UD from the guest bitmap and merge it with the nested-guest bitmap
7821	* here (and avoid doing anything while exporting nested-guest state), but to keep the
7822	* code more flexible if intercepting exceptions become more dynamic in the future we do
7823	* it as part of exporting the nested-guest state.
7824	*/
7825	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
7826
7827	/*
7828	* CR0/CR4 guest/host mask.
7829	*
7830	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest must
7831	* cause VM-exits, so we need to merge them here.
7832	*/
7833	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
7834	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
7835
7836	/*
7837	* Page-fault error-code mask and match.
7838	*
7839	* Although we require unrestricted guest execution (and thereby nested-paging) for
7840	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
7841	* normally intercept #PFs, it might intercept them for debugging purposes.
7842	*
7843	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF filters.
7844	* If the outer guest is intercepting #PFs, we must intercept all #PFs.
7845	*/
7846	uint32_t u32XcptPFMask;
7847	uint32_t u32XcptPFMatch;
7848	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
7849	{
7850	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
7851	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
7852	}
7853	else
7854	{
7855	u32XcptPFMask = 0;
7856	u32XcptPFMatch = 0;
7857	}
7858
7859	/*
7860	* Pause-Loop exiting.
7861	*/
7862	/** @todo r=bird: given that both pVM->hm.s.vmx.cPleGapTicks and
7863	* pVM->hm.s.vmx.cPleWindowTicks defaults to zero, I cannot see how
7864	* this will work... */
7865	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
7866	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
7867
7868	/*
7869	* Pending debug exceptions.
7870	* Currently just copy whatever the nested-guest provides us.
7871	*/
7872	uint64_t const uPendingDbgXcpts = pVmcsNstGst->u64GuestPendingDbgXcpts.u;
7873
7874	/*
7875	* I/O Bitmap.
7876	*
7877	* We do not use the I/O bitmap that may be provided by the nested hypervisor as we always
7878	* intercept all I/O port accesses.
7879	*/
7880	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
7881	Assert(!(u32ProcCtls & VMX_PROC_CTLS_USE_IO_BITMAPS));
7882
7883	/*
7884	* VMCS shadowing.
7885	*
7886	* We do not yet expose VMCS shadowing to the guest and thus VMCS shadowing should not be
7887	* enabled while executing the nested-guest.
7888	*/
7889	Assert(!(u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING));
7890
7891	/*
7892	* APIC-access page.
7893	*/
7894	RTHCPHYS HCPhysApicAccess;
7895	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
7896	{
7897	Assert(g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS);
7898	RTGCPHYS const GCPhysApicAccess = pVmcsNstGst->u64AddrApicAccess.u;
7899
7900	/** @todo NSTVMX: This is not really correct but currently is required to make
7901	* things work. We need to re-enable the page handler when we fallback to
7902	* IEM execution of the nested-guest! */
7903	PGMHandlerPhysicalPageTempOff(pVM, GCPhysApicAccess, GCPhysApicAccess);
7904
7905	void *pvPage;
7906	PGMPAGEMAPLOCK PgLockApicAccess;
7907	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysApicAccess, &pvPage, &PgLockApicAccess);
7908	if (RT_SUCCESS(rc))
7909	{
7910	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysApicAccess, &HCPhysApicAccess);
7911	AssertMsgRCReturn(rc, ("Failed to get host-physical address for APIC-access page at %#RGp\n", GCPhysApicAccess), rc);
7912
7913	/** @todo Handle proper releasing of page-mapping lock later. */
7914	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &PgLockApicAccess);
7915	}
7916	else
7917	return rc;
7918	}
7919	else
7920	HCPhysApicAccess = 0;
7921
7922	/*
7923	* Virtual-APIC page and TPR threshold.
7924	*/
7925	RTHCPHYS HCPhysVirtApic;
7926	uint32_t u32TprThreshold;
7927	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
7928	{
7929	Assert(g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW);
7930	RTGCPHYS const GCPhysVirtApic = pVmcsNstGst->u64AddrVirtApic.u;
7931
7932	void *pvPage;
7933	PGMPAGEMAPLOCK PgLockVirtApic;
7934	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysVirtApic, &pvPage, &PgLockVirtApic);
7935	if (RT_SUCCESS(rc))
7936	{
7937	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysVirtApic, &HCPhysVirtApic);
7938	AssertMsgRCReturn(rc, ("Failed to get host-physical address for virtual-APIC page at %#RGp\n", GCPhysVirtApic), rc);
7939
7940	/** @todo Handle proper releasing of page-mapping lock later. */
7941	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &PgLockVirtApic);
7942	}
7943	else
7944	return rc;
7945
7946	u32TprThreshold = pVmcsNstGst->u32TprThreshold;
7947	}
7948	else
7949	{
7950	HCPhysVirtApic = 0;
7951	u32TprThreshold = 0;
7952
7953	/*
7954	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
7955	* used by the nested hypervisor. Preventing MMIO accesses to the physical APIC will
7956	* be taken care of by EPT/shadow paging.
7957	*/
7958	if (pVM->hmr0.s.fAllow64BitGuests)
7959	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
7960	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
7961	}
7962
7963	/*
7964	* Validate basic assumptions.
7965	*/
7966	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hmr0.s.vmx.VmcsInfoNstGst;
7967	Assert(VM_IS_VMX_UNRESTRICTED_GUEST(pVM));
7968	Assert(g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
7969	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
7970
7971	/*
7972	* Commit it to the nested-guest VMCS.
7973	*/
7974	int rc = VINF_SUCCESS;
7975	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
7976	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
7977	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
7978	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
7979	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
7980	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
7981	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
7982	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
7983	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
7984	rc \|= VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
7985	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
7986	rc \|= VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
7987	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
7988	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
7989	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
7990	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
7991	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
7992	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
7993	{
7994	Assert(g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
7995	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
7996	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
7997	}
7998	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
7999	{
8000	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
8001	rc \|= VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
8002	}
8003	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
8004	rc \|= VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
8005	rc \|= VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, uPendingDbgXcpts);
8006	AssertRC(rc);
8007
8008	/*
8009	* Update the nested-guest VMCS cache.
8010	*/
8011	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
8012	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
8013	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
8014	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
8015	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
8016	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
8017	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
8018	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
8019	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
8020
8021	/*
8022	* We need to flush the TLB if we are switching the APIC-access page address.
8023	* See Intel spec. 28.3.3.4 "Guidelines for Use of the INVEPT Instruction".
8024	*/
8025	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
8026	VCPU_2_VMXSTATE(pVCpu).vmx.fSwitchedNstGstFlushTlb = true;
8027
8028	/*
8029	* MSR bitmap.
8030	*
8031	* The MSR bitmap address has already been initialized while setting up the nested-guest
8032	* VMCS, here we need to merge the MSR bitmaps.
8033	*/
8034	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
8035	vmxHCMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
8036
8037	return VINF_SUCCESS;
8038	}
8039	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
8040
8041
8042	/**
8043	* Runs the guest code using hardware-assisted VMX the normal way.
8044	*
8045	* @returns VBox status code.
8046	* @param pVCpu The cross context virtual CPU structure.
8047	* @param pcLoops Pointer to the number of executed loops.
8048	*/
8049	static VBOXSTRICTRC vmxHCRunGuestCodeNormal(PVMCPUCC pVCpu, uint32_t *pcLoops)
8050	{
8051	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
8052	Assert(pcLoops);
8053	Assert(*pcLoops <= cMaxResumeLoops);
8054	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8055
8056	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8057	/*
8058	* Switch to the guest VMCS as we may have transitioned from executing the nested-guest
8059	* without leaving ring-0. Otherwise, if we came from ring-3 we would have loaded the
8060	* guest VMCS while entering the VMX ring-0 session.
8061	*/
8062	if (pVCpu->hmr0.s.vmx.fSwitchedToNstGstVmcs)
8063	{
8064	int rc = vmxHCSwitchToGstOrNstGstVmcs(pVCpu, false /* fSwitchToNstGstVmcs */);
8065	if (RT_SUCCESS(rc))
8066	{ /* likely */ }
8067	else
8068	{
8069	LogRelFunc(("Failed to switch to the guest VMCS. rc=%Rrc\n", rc));
8070	return rc;
8071	}
8072	}
8073	#endif
8074
8075	VMXTRANSIENT VmxTransient;
8076	RT_ZERO(VmxTransient);
8077	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8078
8079	/* Paranoia. */
8080	Assert(VmxTransient.pVmcsInfo == &pVCpu->hmr0.s.vmx.VmcsInfo);
8081
8082	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8083	for (;;)
8084	{
8085	Assert(!HMR0SuspendPending());
8086	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8087	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatEntry, x);
8088
8089	/*
8090	* Preparatory work for running nested-guest code, this may force us to
8091	* return to ring-3.
8092	*
8093	* Warning! This bugger disables interrupts on VINF_SUCCESS!
8094	*/
8095	rcStrict = vmxHCPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
8096	if (rcStrict != VINF_SUCCESS)
8097	break;
8098
8099	/* Interrupts are disabled at this point! */
8100	vmxHCPreRunGuestCommitted(pVCpu, &VmxTransient);
8101	int rcRun = vmxHCRunGuest(pVCpu, &VmxTransient);
8102	vmxHCPostRunGuest(pVCpu, &VmxTransient, rcRun);
8103	/* Interrupts are re-enabled at this point! */
8104
8105	/*
8106	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
8107	*/
8108	if (RT_SUCCESS(rcRun))
8109	{ /* very likely */ }
8110	else
8111	{
8112	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatPreExit, x);
8113	vmxHCReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
8114	return rcRun;
8115	}
8116
8117	/*
8118	* Profile the VM-exit.
8119	*/
8120	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8121	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitAll);
8122	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).aStatExitReason[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8123	STAM_PROFILE_ADV_STOP_START(&VCPU_2_VMXSTATS(pVCpu).StatPreExit, &VCPU_2_VMXSTATS(pVCpu).StatExitHandling, x);
8124	HMVMX_START_EXIT_DISPATCH_PROF();
8125
8126	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
8127
8128	/*
8129	* Handle the VM-exit.
8130	*/
8131	#ifdef HMVMX_USE_FUNCTION_TABLE
8132	rcStrict = g_aVMExitHandlers[VmxTransient.uExitReason].pfn(pVCpu, &VmxTransient);
8133	#else
8134	rcStrict = vmxHCHandleExit(pVCpu, &VmxTransient);
8135	#endif
8136	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitHandling, x);
8137	if (rcStrict == VINF_SUCCESS)
8138	{
8139	if (++(*pcLoops) <= cMaxResumeLoops)
8140	continue;
8141	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchMaxResumeLoops);
8142	rcStrict = VINF_EM_RAW_INTERRUPT;
8143	}
8144	break;
8145	}
8146
8147	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatEntry, x);
8148	return rcStrict;
8149	}
8150
8151
8152	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8153	/**
8154	* Runs the nested-guest code using hardware-assisted VMX.
8155	*
8156	* @returns VBox status code.
8157	* @param pVCpu The cross context virtual CPU structure.
8158	* @param pcLoops Pointer to the number of executed loops.
8159	*
8160	* @sa vmxHCRunGuestCodeNormal.
8161	*/
8162	static VBOXSTRICTRC vmxHCRunGuestCodeNested(PVMCPUCC pVCpu, uint32_t *pcLoops)
8163	{
8164	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
8165	Assert(pcLoops);
8166	Assert(*pcLoops <= cMaxResumeLoops);
8167	Assert(CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8168
8169	/*
8170	* Switch to the nested-guest VMCS as we may have transitioned from executing the
8171	* guest without leaving ring-0. Otherwise, if we came from ring-3 we would have
8172	* loaded the nested-guest VMCS while entering the VMX ring-0 session.
8173	*/
8174	if (!pVCpu->hmr0.s.vmx.fSwitchedToNstGstVmcs)
8175	{
8176	int rc = vmxHCSwitchToGstOrNstGstVmcs(pVCpu, true /* fSwitchToNstGstVmcs */);
8177	if (RT_SUCCESS(rc))
8178	{ /* likely */ }
8179	else
8180	{
8181	LogRelFunc(("Failed to switch to the nested-guest VMCS. rc=%Rrc\n", rc));
8182	return rc;
8183	}
8184	}
8185
8186	VMXTRANSIENT VmxTransient;
8187	RT_ZERO(VmxTransient);
8188	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8189	VmxTransient.fIsNestedGuest = true;
8190
8191	/* Paranoia. */
8192	Assert(VmxTransient.pVmcsInfo == &pVCpu->hmr0.s.vmx.VmcsInfoNstGst);
8193
8194	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8195	for (;;)
8196	{
8197	Assert(!HMR0SuspendPending());
8198	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8199	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatEntry, x);
8200
8201	/*
8202	* Preparatory work for running guest code, this may force us to
8203	* return to ring-3.
8204	*
8205	* Warning! This bugger disables interrupts on VINF_SUCCESS!
8206	*/
8207	rcStrict = vmxHCPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
8208	if (rcStrict != VINF_SUCCESS)
8209	break;
8210
8211	/* Interrupts are disabled at this point! */
8212	vmxHCPreRunGuestCommitted(pVCpu, &VmxTransient);
8213	int rcRun = vmxHCRunGuest(pVCpu, &VmxTransient);
8214	vmxHCPostRunGuest(pVCpu, &VmxTransient, rcRun);
8215	/* Interrupts are re-enabled at this point! */
8216
8217	/*
8218	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
8219	*/
8220	if (RT_SUCCESS(rcRun))
8221	{ /* very likely */ }
8222	else
8223	{
8224	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatPreExit, x);
8225	vmxHCReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
8226	return rcRun;
8227	}
8228
8229	/*
8230	* Profile the VM-exit.
8231	*/
8232	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8233	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitAll);
8234	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatNestedExitAll);
8235	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).aStatNestedExitReason[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8236	STAM_PROFILE_ADV_STOP_START(&VCPU_2_VMXSTATS(pVCpu).StatPreExit, &VCPU_2_VMXSTATS(pVCpu).StatExitHandling, x);
8237	HMVMX_START_EXIT_DISPATCH_PROF();
8238
8239	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
8240
8241	/*
8242	* Handle the VM-exit.
8243	*/
8244	rcStrict = vmxHCHandleExitNested(pVCpu, &VmxTransient);
8245	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitHandling, x);
8246	if (rcStrict == VINF_SUCCESS)
8247	{
8248	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8249	{
8250	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchNstGstVmexit);
8251	rcStrict = VINF_VMX_VMEXIT;
8252	}
8253	else
8254	{
8255	if (++(*pcLoops) <= cMaxResumeLoops)
8256	continue;
8257	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchMaxResumeLoops);
8258	rcStrict = VINF_EM_RAW_INTERRUPT;
8259	}
8260	}
8261	else
8262	Assert(rcStrict != VINF_VMX_VMEXIT);
8263	break;
8264	}
8265
8266	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatEntry, x);
8267	return rcStrict;
8268	}
8269	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
8270
8271
8272	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8273	* probes.
8274	*
8275	* The following few functions and associated structure contains the bloat
8276	* necessary for providing detailed debug events and dtrace probes as well as
8277	* reliable host side single stepping. This works on the principle of
8278	* "subclassing" the normal execution loop and workers. We replace the loop
8279	* method completely and override selected helpers to add necessary adjustments
8280	* to their core operation.
8281	*
8282	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8283	* any performance for debug and analysis features.
8284	*
8285	* @{
8286	*/
8287
8288	/**
8289	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
8290	* the debug run loop.
8291	*/
8292	typedef struct VMXRUNDBGSTATE
8293	{
8294	/** The RIP we started executing at. This is for detecting that we stepped. */
8295	uint64_t uRipStart;
8296	/** The CS we started executing with. */
8297	uint16_t uCsStart;
8298
8299	/** Whether we've actually modified the 1st execution control field. */
8300	bool fModifiedProcCtls : 1;
8301	/** Whether we've actually modified the 2nd execution control field. */
8302	bool fModifiedProcCtls2 : 1;
8303	/** Whether we've actually modified the exception bitmap. */
8304	bool fModifiedXcptBitmap : 1;
8305
8306	/** We desire the modified the CR0 mask to be cleared. */
8307	bool fClearCr0Mask : 1;
8308	/** We desire the modified the CR4 mask to be cleared. */
8309	bool fClearCr4Mask : 1;
8310	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8311	uint32_t fCpe1Extra;
8312	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8313	uint32_t fCpe1Unwanted;
8314	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8315	uint32_t fCpe2Extra;
8316	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
8317	uint32_t bmXcptExtra;
8318	/** The sequence number of the Dtrace provider settings the state was
8319	* configured against. */
8320	uint32_t uDtraceSettingsSeqNo;
8321	/** VM-exits to check (one bit per VM-exit). */
8322	uint32_t bmExitsToCheck[3];
8323
8324	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8325	uint32_t fProcCtlsInitial;
8326	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8327	uint32_t fProcCtls2Initial;
8328	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8329	uint32_t bmXcptInitial;
8330	} VMXRUNDBGSTATE;
8331	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
8332	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
8333
8334
8335	/**
8336	* Initializes the VMXRUNDBGSTATE structure.
8337	*
8338	* @param pVCpu The cross context virtual CPU structure of the
8339	* calling EMT.
8340	* @param pVmxTransient The VMX-transient structure.
8341	* @param pDbgState The debug state to initialize.
8342	*/
8343	static void vmxHCRunDebugStateInit(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
8344	{
8345	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
8346	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
8347
8348	pDbgState->fModifiedProcCtls = false;
8349	pDbgState->fModifiedProcCtls2 = false;
8350	pDbgState->fModifiedXcptBitmap = false;
8351	pDbgState->fClearCr0Mask = false;
8352	pDbgState->fClearCr4Mask = false;
8353	pDbgState->fCpe1Extra = 0;
8354	pDbgState->fCpe1Unwanted = 0;
8355	pDbgState->fCpe2Extra = 0;
8356	pDbgState->bmXcptExtra = 0;
8357	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
8358	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
8359	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
8360	}
8361
8362
8363	/**
8364	* Updates the VMSC fields with changes requested by @a pDbgState.
8365	*
8366	* This is performed after vmxHCPreRunGuestDebugStateUpdate as well
8367	* immediately before executing guest code, i.e. when interrupts are disabled.
8368	* We don't check status codes here as we cannot easily assert or return in the
8369	* latter case.
8370	*
8371	* @param pVCpu The cross context virtual CPU structure.
8372	* @param pVmxTransient The VMX-transient structure.
8373	* @param pDbgState The debug state.
8374	*/
8375	static void vmxHCPreRunGuestDebugStateApply(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
8376	{
8377	/*
8378	* Ensure desired flags in VMCS control fields are set.
8379	* (Ignoring write failure here, as we're committed and it's just debug extras.)
8380	*
8381	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
8382	* there should be no stale data in pCtx at this point.
8383	*/
8384	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8385	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
8386	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
8387	{
8388	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
8389	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
8390	VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8391	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
8392	pDbgState->fModifiedProcCtls = true;
8393	}
8394
8395	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
8396	{
8397	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
8398	VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
8399	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
8400	pDbgState->fModifiedProcCtls2 = true;
8401	}
8402
8403	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
8404	{
8405	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
8406	VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
8407	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
8408	pDbgState->fModifiedXcptBitmap = true;
8409	}
8410
8411	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
8412	{
8413	pVmcsInfo->u64Cr0Mask = 0;
8414	VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_MASK, 0);
8415	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
8416	}
8417
8418	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
8419	{
8420	pVmcsInfo->u64Cr4Mask = 0;
8421	VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR4_MASK, 0);
8422	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
8423	}
8424
8425	NOREF(pVCpu);
8426	}
8427
8428
8429	/**
8430	* Restores VMCS fields that were changed by vmxHCPreRunGuestDebugStateApply for
8431	* re-entry next time around.
8432	*
8433	* @returns Strict VBox status code (i.e. informational status codes too).
8434	* @param pVCpu The cross context virtual CPU structure.
8435	* @param pVmxTransient The VMX-transient structure.
8436	* @param pDbgState The debug state.
8437	* @param rcStrict The return code from executing the guest using single
8438	* stepping.
8439	*/
8440	static VBOXSTRICTRC vmxHCRunDebugStateRevert(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
8441	VBOXSTRICTRC rcStrict)
8442	{
8443	/*
8444	* Restore VM-exit control settings as we may not reenter this function the
8445	* next time around.
8446	*/
8447	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8448
8449	/* We reload the initial value, trigger what we can of recalculations the
8450	next time around. From the looks of things, that's all that's required atm. */
8451	if (pDbgState->fModifiedProcCtls)
8452	{
8453	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
8454	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in vmxHCLeave */
8455	int rc2 = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
8456	AssertRC(rc2);
8457	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
8458	}
8459
8460	/* We're currently the only ones messing with this one, so just restore the
8461	cached value and reload the field. */
8462	if ( pDbgState->fModifiedProcCtls2
8463	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
8464	{
8465	int rc2 = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
8466	AssertRC(rc2);
8467	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
8468	}
8469
8470	/* If we've modified the exception bitmap, we restore it and trigger
8471	reloading and partial recalculation the next time around. */
8472	if (pDbgState->fModifiedXcptBitmap)
8473	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
8474
8475	return rcStrict;
8476	}
8477
8478
8479	/**
8480	* Configures VM-exit controls for current DBGF and DTrace settings.
8481	*
8482	* This updates @a pDbgState and the VMCS execution control fields to reflect
8483	* the necessary VM-exits demanded by DBGF and DTrace.
8484	*
8485	* @param pVCpu The cross context virtual CPU structure.
8486	* @param pVmxTransient The VMX-transient structure. May update
8487	* fUpdatedTscOffsettingAndPreemptTimer.
8488	* @param pDbgState The debug state.
8489	*/
8490	static void vmxHCPreRunGuestDebugStateUpdate(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
8491	{
8492	/*
8493	* Take down the dtrace serial number so we can spot changes.
8494	*/
8495	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
8496	ASMCompilerBarrier();
8497
8498	/*
8499	* We'll rebuild most of the middle block of data members (holding the
8500	* current settings) as we go along here, so start by clearing it all.
8501	*/
8502	pDbgState->bmXcptExtra = 0;
8503	pDbgState->fCpe1Extra = 0;
8504	pDbgState->fCpe1Unwanted = 0;
8505	pDbgState->fCpe2Extra = 0;
8506	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
8507	pDbgState->bmExitsToCheck[i] = 0;
8508
8509	/*
8510	* Software interrupts (INT XXh) - no idea how to trigger these...
8511	*/
8512	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8513	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
8514	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
8515	{
8516	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8517	}
8518
8519	/*
8520	* INT3 breakpoints - triggered by #BP exceptions.
8521	*/
8522	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
8523	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
8524
8525	/*
8526	* Exception bitmap and XCPT events+probes.
8527	*/
8528	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
8529	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
8530	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
8531
8532	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
8533	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
8534	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
8535	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
8536	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
8537	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
8538	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
8539	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
8540	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
8541	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
8542	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
8543	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
8544	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
8545	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
8546	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
8547	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
8548	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
8549	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
8550
8551	if (pDbgState->bmXcptExtra)
8552	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8553
8554	/*
8555	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
8556	*
8557	* Note! This is the reverse of what vmxHCHandleExitDtraceEvents does.
8558	* So, when adding/changing/removing please don't forget to update it.
8559	*
8560	* Some of the macros are picking up local variables to save horizontal space,
8561	* (being able to see it in a table is the lesser evil here).
8562	*/
8563	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
8564	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
8565	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
8566	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
8567	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8568	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
8569	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
8570	} else do { } while (0)
8571	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
8572	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8573	{ \
8574	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
8575	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
8576	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
8577	} else do { } while (0)
8578	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
8579	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8580	{ \
8581	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
8582	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
8583	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
8584	} else do { } while (0)
8585	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
8586	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8587	{ \
8588	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
8589	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
8590	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
8591	} else do { } while (0)
8592
8593	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
8594	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
8595	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
8596	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
8597	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
8598
8599	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
8600	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
8601	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
8602	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
8603	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
8604	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
8605	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
8606	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
8607	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
8608	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
8609	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
8610	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
8611	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
8612	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
8613	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
8614	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
8615	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
8616	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
8617	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
8618	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
8619	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
8620	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
8621	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
8622	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
8623	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
8624	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
8625	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
8626	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
8627	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
8628	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
8629	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
8630	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
8631	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
8632	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
8633	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
8634	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
8635
8636	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
8637	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
8638	{
8639	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
8640	\| CPUMCTX_EXTRN_APIC_TPR);
8641	AssertRC(rc);
8642
8643	#if 0 /** @todo fix me */
8644	pDbgState->fClearCr0Mask = true;
8645	pDbgState->fClearCr4Mask = true;
8646	#endif
8647	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
8648	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
8649	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
8650	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
8651	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
8652	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
8653	require clearing here and in the loop if we start using it. */
8654	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
8655	}
8656	else
8657	{
8658	if (pDbgState->fClearCr0Mask)
8659	{
8660	pDbgState->fClearCr0Mask = false;
8661	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_CR0);
8662	}
8663	if (pDbgState->fClearCr4Mask)
8664	{
8665	pDbgState->fClearCr4Mask = false;
8666	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_CR4);
8667	}
8668	}
8669	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
8670	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
8671
8672	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
8673	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
8674	{
8675	/** @todo later, need to fix handler as it assumes this won't usually happen. */
8676	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
8677	}
8678	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
8679	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
8680
8681	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
8682	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
8683	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
8684	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
8685	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
8686	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
8687	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
8688	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
8689	#if 0 /** @todo too slow, fix handler. */
8690	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
8691	#endif
8692	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
8693
8694	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
8695	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
8696	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
8697	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
8698	{
8699	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
8700	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
8701	}
8702	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
8703	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
8704	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
8705	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
8706
8707	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
8708	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
8709	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
8710	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
8711	{
8712	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
8713	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
8714	}
8715	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
8716	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
8717	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
8718	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
8719
8720	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
8721	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
8722	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
8723	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
8724	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
8725	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
8726	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
8727	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
8728	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
8729	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
8730	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
8731	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
8732	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
8733	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
8734	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
8735	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
8736	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
8737	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
8738	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
8739	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
8740	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
8741	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
8742
8743	#undef IS_EITHER_ENABLED
8744	#undef SET_ONLY_XBM_IF_EITHER_EN
8745	#undef SET_CPE1_XBM_IF_EITHER_EN
8746	#undef SET_CPEU_XBM_IF_EITHER_EN
8747	#undef SET_CPE2_XBM_IF_EITHER_EN
8748
8749	/*
8750	* Sanitize the control stuff.
8751	*/
8752	pDbgState->fCpe2Extra &= g_HmMsrs.u.vmx.ProcCtls2.n.allowed1;
8753	if (pDbgState->fCpe2Extra)
8754	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
8755	pDbgState->fCpe1Extra &= g_HmMsrs.u.vmx.ProcCtls.n.allowed1;
8756	pDbgState->fCpe1Unwanted &= ~g_HmMsrs.u.vmx.ProcCtls.n.allowed0;
8757	if (pVCpu->hmr0.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
8758	{
8759	pVCpu->hmr0.s.fDebugWantRdTscExit ^= true;
8760	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
8761	}
8762
8763	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
8764	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
8765	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
8766	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
8767	}
8768
8769
8770	/**
8771	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
8772	* appropriate.
8773	*
8774	* The caller has checked the VM-exit against the
8775	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
8776	* already, so we don't have to do that either.
8777	*
8778	* @returns Strict VBox status code (i.e. informational status codes too).
8779	* @param pVCpu The cross context virtual CPU structure.
8780	* @param pVmxTransient The VMX-transient structure.
8781	* @param uExitReason The VM-exit reason.
8782	*
8783	* @remarks The name of this function is displayed by dtrace, so keep it short
8784	* and to the point. No longer than 33 chars long, please.
8785	*/
8786	static VBOXSTRICTRC vmxHCHandleExitDtraceEvents(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
8787	{
8788	/*
8789	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
8790	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
8791	*
8792	* Note! This is the reverse operation of what vmxHCPreRunGuestDebugStateUpdate
8793	* does. Must add/change/remove both places. Same ordering, please.
8794	*
8795	* Added/removed events must also be reflected in the next section
8796	* where we dispatch dtrace events.
8797	*/
8798	bool fDtrace1 = false;
8799	bool fDtrace2 = false;
8800	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
8801	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
8802	uint32_t uEventArg = 0;
8803	#define SET_EXIT(a_EventSubName) \
8804	do { \
8805	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
8806	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
8807	} while (0)
8808	#define SET_BOTH(a_EventSubName) \
8809	do { \
8810	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
8811	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
8812	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
8813	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
8814	} while (0)
8815	switch (uExitReason)
8816	{
8817	case VMX_EXIT_MTF:
8818	return vmxHCExitMtf(pVCpu, pVmxTransient);
8819
8820	case VMX_EXIT_XCPT_OR_NMI:
8821	{
8822	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
8823	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
8824	{
8825	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
8826	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
8827	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
8828	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
8829	{
8830	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
8831	{
8832	vmxHCReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
8833	uEventArg = pVmxTransient->uExitIntErrorCode;
8834	}
8835	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
8836	switch (enmEvent1)
8837	{
8838	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
8839	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
8840	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
8841	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
8842	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
8843	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
8844	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
8845	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
8846	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
8847	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
8848	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
8849	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
8850	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
8851	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
8852	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
8853	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
8854	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
8855	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
8856	default: break;
8857	}
8858	}
8859	else
8860	AssertFailed();
8861	break;
8862
8863	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
8864	uEventArg = idxVector;
8865	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
8866	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
8867	break;
8868	}
8869	break;
8870	}
8871
8872	case VMX_EXIT_TRIPLE_FAULT:
8873	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
8874	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
8875	break;
8876	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
8877	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
8878	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
8879	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
8880	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
8881
8882	/* Instruction specific VM-exits: */
8883	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
8884	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
8885	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
8886	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
8887	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
8888	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
8889	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
8890	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
8891	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
8892	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
8893	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
8894	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
8895	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
8896	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
8897	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
8898	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
8899	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
8900	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
8901	case VMX_EXIT_MOV_CRX:
8902	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
8903	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
8904	SET_BOTH(CRX_READ);
8905	else
8906	SET_BOTH(CRX_WRITE);
8907	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
8908	break;
8909	case VMX_EXIT_MOV_DRX:
8910	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
8911	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
8912	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
8913	SET_BOTH(DRX_READ);
8914	else
8915	SET_BOTH(DRX_WRITE);
8916	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
8917	break;
8918	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
8919	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
8920	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
8921	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
8922	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
8923	case VMX_EXIT_GDTR_IDTR_ACCESS:
8924	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
8925	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
8926	{
8927	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
8928	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
8929	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
8930	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
8931	}
8932	break;
8933
8934	case VMX_EXIT_LDTR_TR_ACCESS:
8935	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
8936	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
8937	{
8938	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
8939	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
8940	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
8941	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
8942	}
8943	break;
8944
8945	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
8946	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
8947	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
8948	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
8949	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
8950	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
8951	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
8952	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
8953	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
8954	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
8955	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
8956
8957	/* Events that aren't relevant at this point. */
8958	case VMX_EXIT_EXT_INT:
8959	case VMX_EXIT_INT_WINDOW:
8960	case VMX_EXIT_NMI_WINDOW:
8961	case VMX_EXIT_TPR_BELOW_THRESHOLD:
8962	case VMX_EXIT_PREEMPT_TIMER:
8963	case VMX_EXIT_IO_INSTR:
8964	break;
8965
8966	/* Errors and unexpected events. */
8967	case VMX_EXIT_INIT_SIGNAL:
8968	case VMX_EXIT_SIPI:
8969	case VMX_EXIT_IO_SMI:
8970	case VMX_EXIT_SMI:
8971	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
8972	case VMX_EXIT_ERR_MSR_LOAD:
8973	case VMX_EXIT_ERR_MACHINE_CHECK:
8974	case VMX_EXIT_PML_FULL:
8975	case VMX_EXIT_VIRTUALIZED_EOI:
8976	break;
8977
8978	default:
8979	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
8980	break;
8981	}
8982	#undef SET_BOTH
8983	#undef SET_EXIT
8984
8985	/*
8986	* Dtrace tracepoints go first. We do them here at once so we don't
8987	* have to copy the guest state saving and stuff a few dozen times.
8988	* Down side is that we've got to repeat the switch, though this time
8989	* we use enmEvent since the probes are a subset of what DBGF does.
8990	*/
8991	if (fDtrace1 \|\| fDtrace2)
8992	{
8993	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
8994	vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8995	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8996	switch (enmEvent1)
8997	{
8998	/** @todo consider which extra parameters would be helpful for each probe. */
8999	case DBGFEVENT_END: break;
9000	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
9001	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
9002	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
9003	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
9004	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
9005	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
9006	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
9007	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
9008	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
9009	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
9010	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
9011	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
9012	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
9013	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
9014	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
9015	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
9016	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
9017	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
9018	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9019	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9020	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
9021	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
9022	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
9023	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
9024	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
9025	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
9026	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
9027	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9028	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9029	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9030	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9031	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9032	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
9033	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9034	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
9035	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
9036	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
9037	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
9038	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
9039	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
9040	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
9041	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
9042	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
9043	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
9044	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
9045	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
9046	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
9047	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
9048	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
9049	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
9050	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
9051	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
9052	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
9053	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
9054	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
9055	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
9056	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
9057	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
9058	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
9059	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
9060	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
9061	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
9062	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
9063	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
9064	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
9065	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
9066	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9067	}
9068	switch (enmEvent2)
9069	{
9070	/** @todo consider which extra parameters would be helpful for each probe. */
9071	case DBGFEVENT_END: break;
9072	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
9073	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9074	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
9075	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
9076	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
9077	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
9078	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
9079	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
9080	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
9081	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9082	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9083	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9084	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9085	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9086	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
9087	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9088	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
9089	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
9090	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
9091	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
9092	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
9093	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
9094	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
9095	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
9096	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
9097	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
9098	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
9099	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
9100	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
9101	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
9102	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
9103	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
9104	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
9105	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
9106	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
9107	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
9108	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
9109	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
9110	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
9111	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
9112	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
9113	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
9114	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
9115	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
9116	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
9117	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
9118	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
9119	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
9120	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
9121	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
9122	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
9123	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
9124	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9125	}
9126	}
9127
9128	/*
9129	* Fire of the DBGF event, if enabled (our check here is just a quick one,
9130	* the DBGF call will do a full check).
9131	*
9132	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9133	* Note! If we have to events, we prioritize the first, i.e. the instruction
9134	* one, in order to avoid event nesting.
9135	*/
9136	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
9137	if ( enmEvent1 != DBGFEVENT_END
9138	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9139	{
9140	vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
9141	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
9142	if (rcStrict != VINF_SUCCESS)
9143	return rcStrict;
9144	}
9145	else if ( enmEvent2 != DBGFEVENT_END
9146	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9147	{
9148	vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
9149	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
9150	if (rcStrict != VINF_SUCCESS)
9151	return rcStrict;
9152	}
9153
9154	return VINF_SUCCESS;
9155	}
9156
9157
9158	/**
9159	* Single-stepping VM-exit filtering.
9160	*
9161	* This is preprocessing the VM-exits and deciding whether we've gotten far
9162	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9163	* handling is performed.
9164	*
9165	* @returns Strict VBox status code (i.e. informational status codes too).
9166	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
9167	* @param pVmxTransient The VMX-transient structure.
9168	* @param pDbgState The debug state.
9169	*/
9170	DECLINLINE(VBOXSTRICTRC) vmxHCRunDebugHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
9171	{
9172	/*
9173	* Expensive (saves context) generic dtrace VM-exit probe.
9174	*/
9175	uint32_t const uExitReason = pVmxTransient->uExitReason;
9176	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9177	{ /* more likely */ }
9178	else
9179	{
9180	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
9181	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
9182	AssertRC(rc);
9183	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
9184	}
9185
9186	#ifdef IN_RING0 /* NMIs should never reach R3. */
9187	/*
9188	* Check for host NMI, just to get that out of the way.
9189	*/
9190	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9191	{ /* normally likely */ }
9192	else
9193	{
9194	vmxHCReadExitIntInfoVmcs(pVCpu, pVmxTransient);
9195	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
9196	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
9197	return vmxHCExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
9198	}
9199	#endif
9200
9201	/*
9202	* Check for single stepping event if we're stepping.
9203	*/
9204	if (VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
9205	{
9206	switch (uExitReason)
9207	{
9208	case VMX_EXIT_MTF:
9209	return vmxHCExitMtf(pVCpu, pVmxTransient);
9210
9211	/* Various events: */
9212	case VMX_EXIT_XCPT_OR_NMI:
9213	case VMX_EXIT_EXT_INT:
9214	case VMX_EXIT_TRIPLE_FAULT:
9215	case VMX_EXIT_INT_WINDOW:
9216	case VMX_EXIT_NMI_WINDOW:
9217	case VMX_EXIT_TASK_SWITCH:
9218	case VMX_EXIT_TPR_BELOW_THRESHOLD:
9219	case VMX_EXIT_APIC_ACCESS:
9220	case VMX_EXIT_EPT_VIOLATION:
9221	case VMX_EXIT_EPT_MISCONFIG:
9222	case VMX_EXIT_PREEMPT_TIMER:
9223
9224	/* Instruction specific VM-exits: */
9225	case VMX_EXIT_CPUID:
9226	case VMX_EXIT_GETSEC:
9227	case VMX_EXIT_HLT:
9228	case VMX_EXIT_INVD:
9229	case VMX_EXIT_INVLPG:
9230	case VMX_EXIT_RDPMC:
9231	case VMX_EXIT_RDTSC:
9232	case VMX_EXIT_RSM:
9233	case VMX_EXIT_VMCALL:
9234	case VMX_EXIT_VMCLEAR:
9235	case VMX_EXIT_VMLAUNCH:
9236	case VMX_EXIT_VMPTRLD:
9237	case VMX_EXIT_VMPTRST:
9238	case VMX_EXIT_VMREAD:
9239	case VMX_EXIT_VMRESUME:
9240	case VMX_EXIT_VMWRITE:
9241	case VMX_EXIT_VMXOFF:
9242	case VMX_EXIT_VMXON:
9243	case VMX_EXIT_MOV_CRX:
9244	case VMX_EXIT_MOV_DRX:
9245	case VMX_EXIT_IO_INSTR:
9246	case VMX_EXIT_RDMSR:
9247	case VMX_EXIT_WRMSR:
9248	case VMX_EXIT_MWAIT:
9249	case VMX_EXIT_MONITOR:
9250	case VMX_EXIT_PAUSE:
9251	case VMX_EXIT_GDTR_IDTR_ACCESS:
9252	case VMX_EXIT_LDTR_TR_ACCESS:
9253	case VMX_EXIT_INVEPT:
9254	case VMX_EXIT_RDTSCP:
9255	case VMX_EXIT_INVVPID:
9256	case VMX_EXIT_WBINVD:
9257	case VMX_EXIT_XSETBV:
9258	case VMX_EXIT_RDRAND:
9259	case VMX_EXIT_INVPCID:
9260	case VMX_EXIT_VMFUNC:
9261	case VMX_EXIT_RDSEED:
9262	case VMX_EXIT_XSAVES:
9263	case VMX_EXIT_XRSTORS:
9264	{
9265	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
9266	AssertRCReturn(rc, rc);
9267	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
9268	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
9269	return VINF_EM_DBG_STEPPED;
9270	break;
9271	}
9272
9273	/* Errors and unexpected events: */
9274	case VMX_EXIT_INIT_SIGNAL:
9275	case VMX_EXIT_SIPI:
9276	case VMX_EXIT_IO_SMI:
9277	case VMX_EXIT_SMI:
9278	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9279	case VMX_EXIT_ERR_MSR_LOAD:
9280	case VMX_EXIT_ERR_MACHINE_CHECK:
9281	case VMX_EXIT_PML_FULL:
9282	case VMX_EXIT_VIRTUALIZED_EOI:
9283	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9284	break;
9285
9286	default:
9287	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9288	break;
9289	}
9290	}
9291
9292	/*
9293	* Check for debugger event breakpoints and dtrace probes.
9294	*/
9295	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9296	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9297	{
9298	VBOXSTRICTRC rcStrict = vmxHCHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
9299	if (rcStrict != VINF_SUCCESS)
9300	return rcStrict;
9301	}
9302
9303	/*
9304	* Normal processing.
9305	*/
9306	#ifdef HMVMX_USE_FUNCTION_TABLE
9307	return g_aVMExitHandlers[uExitReason].pfn(pVCpu, pVmxTransient);
9308	#else
9309	return vmxHCHandleExit(pVCpu, pVmxTransient, uExitReason);
9310	#endif
9311	}
9312
9313
9314	/**
9315	* Single steps guest code using hardware-assisted VMX.
9316	*
9317	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
9318	* but single-stepping through the hypervisor debugger.
9319	*
9320	* @returns Strict VBox status code (i.e. informational status codes too).
9321	* @param pVCpu The cross context virtual CPU structure.
9322	* @param pcLoops Pointer to the number of executed loops.
9323	*
9324	* @note Mostly the same as vmxHCRunGuestCodeNormal().
9325	*/
9326	static VBOXSTRICTRC vmxHCRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops)
9327	{
9328	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
9329	Assert(pcLoops);
9330	Assert(*pcLoops <= cMaxResumeLoops);
9331
9332	VMXTRANSIENT VmxTransient;
9333	RT_ZERO(VmxTransient);
9334	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9335
9336	/* Set HMCPU indicators. */
9337	bool const fSavedSingleInstruction = VCPU_2_VMXSTATE(pVCpu).fSingleInstruction;
9338	VCPU_2_VMXSTATE(pVCpu).fSingleInstruction = VCPU_2_VMXSTATE(pVCpu).fSingleInstruction \|\| DBGFIsStepping(pVCpu);
9339	pVCpu->hmr0.s.fDebugWantRdTscExit = false;
9340	pVCpu->hmr0.s.fUsingDebugLoop = true;
9341
9342	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9343	VMXRUNDBGSTATE DbgState;
9344	vmxHCRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
9345	vmxHCPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
9346
9347	/*
9348	* The loop.
9349	*/
9350	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9351	for (;;)
9352	{
9353	Assert(!HMR0SuspendPending());
9354	HMVMX_ASSERT_CPU_SAFE(pVCpu);
9355	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatEntry, x);
9356	bool fStepping = VCPU_2_VMXSTATE(pVCpu).fSingleInstruction;
9357
9358	/* Set up VM-execution controls the next two can respond to. */
9359	vmxHCPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
9360
9361	/*
9362	* Preparatory work for running guest code, this may force us to
9363	* return to ring-3.
9364	*
9365	* Warning! This bugger disables interrupts on VINF_SUCCESS!
9366	*/
9367	rcStrict = vmxHCPreRunGuest(pVCpu, &VmxTransient, fStepping);
9368	if (rcStrict != VINF_SUCCESS)
9369	break;
9370
9371	/* Interrupts are disabled at this point! */
9372	vmxHCPreRunGuestCommitted(pVCpu, &VmxTransient);
9373
9374	/* Override any obnoxious code in the above two calls. */
9375	vmxHCPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
9376
9377	/*
9378	* Finally execute the guest.
9379	*/
9380	int rcRun = vmxHCRunGuest(pVCpu, &VmxTransient);
9381
9382	vmxHCPostRunGuest(pVCpu, &VmxTransient, rcRun);
9383	/* Interrupts are re-enabled at this point! */
9384
9385	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9386	if (RT_SUCCESS(rcRun))
9387	{ /* very likely */ }
9388	else
9389	{
9390	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatPreExit, x);
9391	vmxHCReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
9392	return rcRun;
9393	}
9394
9395	/* Profile the VM-exit. */
9396	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9397	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitAll);
9398	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).aStatExitReason[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9399	STAM_PROFILE_ADV_STOP_START(&VCPU_2_VMXSTATS(pVCpu).StatPreExit, &VCPU_2_VMXSTATS(pVCpu).StatExitHandling, x);
9400	HMVMX_START_EXIT_DISPATCH_PROF();
9401
9402	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
9403
9404	/*
9405	* Handle the VM-exit - we quit earlier on certain VM-exits, see vmxHCHandleExitDebug().
9406	*/
9407	rcStrict = vmxHCRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
9408	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitHandling, x);
9409	if (rcStrict != VINF_SUCCESS)
9410	break;
9411	if (++(*pcLoops) > cMaxResumeLoops)
9412	{
9413	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchMaxResumeLoops);
9414	rcStrict = VINF_EM_RAW_INTERRUPT;
9415	break;
9416	}
9417
9418	/*
9419	* Stepping: Did the RIP change, if so, consider it a single step.
9420	* Otherwise, make sure one of the TFs gets set.
9421	*/
9422	if (fStepping)
9423	{
9424	int rc = vmxHCImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
9425	AssertRC(rc);
9426	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
9427	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
9428	{
9429	rcStrict = VINF_EM_DBG_STEPPED;
9430	break;
9431	}
9432	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_DR7);
9433	}
9434
9435	/*
9436	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
9437	*/
9438	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
9439	vmxHCPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
9440
9441	/* Restore all controls applied by vmxHCPreRunGuestDebugStateApply above. */
9442	rcStrict = vmxHCRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
9443	Assert(rcStrict == VINF_SUCCESS);
9444	}
9445
9446	/*
9447	* Clear the X86_EFL_TF if necessary.
9448	*/
9449	if (pVCpu->hmr0.s.fClearTrapFlag)
9450	{
9451	int rc = vmxHCImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
9452	AssertRC(rc);
9453	pVCpu->hmr0.s.fClearTrapFlag = false;
9454	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
9455	}
9456	/** @todo there seems to be issues with the resume flag when the monitor trap
9457	* flag is pending without being used. Seen early in bios init when
9458	* accessing APIC page in protected mode. */
9459
9460	/* Restore HMCPU indicators. */
9461	pVCpu->hmr0.s.fUsingDebugLoop = false;
9462	pVCpu->hmr0.s.fDebugWantRdTscExit = false;
9463	VCPU_2_VMXSTATE(pVCpu).fSingleInstruction = fSavedSingleInstruction;
9464
9465	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatEntry, x);
9466	return rcStrict;
9467	}
9468	#endif
9469
9470	/** @} */
9471
9472
9473	#ifndef HMVMX_USE_FUNCTION_TABLE
9474	/**
9475	* Handles a guest VM-exit from hardware-assisted VMX execution.
9476	*
9477	* @returns Strict VBox status code (i.e. informational status codes too).
9478	* @param pVCpu The cross context virtual CPU structure.
9479	* @param pVmxTransient The VMX-transient structure.
9480	*/
9481	DECLINLINE(VBOXSTRICTRC) vmxHCHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9482	{
9483	#ifdef DEBUG_ramshankar
9484	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
9485	do { \
9486	if (a_fSave != 0) \
9487	vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL); \
9488	VBOXSTRICTRC rcStrict = a_CallExpr; \
9489	if (a_fSave != 0) \
9490	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST); \
9491	return rcStrict; \
9492	} while (0)
9493	#else
9494	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
9495	#endif
9496	uint32_t const uExitReason = pVmxTransient->uExitReason;
9497	switch (uExitReason)
9498	{
9499	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, vmxHCExitEptMisconfig(pVCpu, pVmxTransient));
9500	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, vmxHCExitEptViolation(pVCpu, pVmxTransient));
9501	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, vmxHCExitIoInstr(pVCpu, pVmxTransient));
9502	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, vmxHCExitCpuid(pVCpu, pVmxTransient));
9503	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, vmxHCExitRdtsc(pVCpu, pVmxTransient));
9504	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, vmxHCExitRdtscp(pVCpu, pVmxTransient));
9505	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, vmxHCExitApicAccess(pVCpu, pVmxTransient));
9506	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, vmxHCExitXcptOrNmi(pVCpu, pVmxTransient));
9507	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, vmxHCExitMovCRx(pVCpu, pVmxTransient));
9508	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, vmxHCExitExtInt(pVCpu, pVmxTransient));
9509	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, vmxHCExitIntWindow(pVCpu, pVmxTransient));
9510	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, vmxHCExitTprBelowThreshold(pVCpu, pVmxTransient));
9511	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, vmxHCExitMwait(pVCpu, pVmxTransient));
9512	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, vmxHCExitMonitor(pVCpu, pVmxTransient));
9513	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, vmxHCExitTaskSwitch(pVCpu, pVmxTransient));
9514	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, vmxHCExitPreemptTimer(pVCpu, pVmxTransient));
9515	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, vmxHCExitRdmsr(pVCpu, pVmxTransient));
9516	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, vmxHCExitWrmsr(pVCpu, pVmxTransient));
9517	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, vmxHCExitVmcall(pVCpu, pVmxTransient));
9518	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, vmxHCExitMovDRx(pVCpu, pVmxTransient));
9519	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, vmxHCExitHlt(pVCpu, pVmxTransient));
9520	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, vmxHCExitInvd(pVCpu, pVmxTransient));
9521	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, vmxHCExitInvlpg(pVCpu, pVmxTransient));
9522	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, vmxHCExitMtf(pVCpu, pVmxTransient));
9523	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, vmxHCExitPause(pVCpu, pVmxTransient));
9524	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, vmxHCExitWbinvd(pVCpu, pVmxTransient));
9525	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, vmxHCExitXsetbv(pVCpu, pVmxTransient));
9526	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, vmxHCExitInvpcid(pVCpu, pVmxTransient));
9527	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, vmxHCExitGetsec(pVCpu, pVmxTransient));
9528	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, vmxHCExitRdpmc(pVCpu, pVmxTransient));
9529	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9530	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, vmxHCExitVmclear(pVCpu, pVmxTransient));
9531	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, vmxHCExitVmlaunch(pVCpu, pVmxTransient));
9532	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, vmxHCExitVmptrld(pVCpu, pVmxTransient));
9533	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, vmxHCExitVmptrst(pVCpu, pVmxTransient));
9534	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, vmxHCExitVmread(pVCpu, pVmxTransient));
9535	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, vmxHCExitVmwrite(pVCpu, pVmxTransient));
9536	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, vmxHCExitVmresume(pVCpu, pVmxTransient));
9537	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, vmxHCExitVmxoff(pVCpu, pVmxTransient));
9538	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, vmxHCExitVmxon(pVCpu, pVmxTransient));
9539	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, vmxHCExitInvvpid(pVCpu, pVmxTransient));
9540	case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, vmxHCExitSetPendingXcptUD(pVCpu, pVmxTransient));
9541	#else
9542	case VMX_EXIT_VMCLEAR:
9543	case VMX_EXIT_VMLAUNCH:
9544	case VMX_EXIT_VMPTRLD:
9545	case VMX_EXIT_VMPTRST:
9546	case VMX_EXIT_VMREAD:
9547	case VMX_EXIT_VMRESUME:
9548	case VMX_EXIT_VMWRITE:
9549	case VMX_EXIT_VMXOFF:
9550	case VMX_EXIT_VMXON:
9551	case VMX_EXIT_INVVPID:
9552	case VMX_EXIT_INVEPT:
9553	return vmxHCExitSetPendingXcptUD(pVCpu, pVmxTransient);
9554	#endif
9555
9556	case VMX_EXIT_TRIPLE_FAULT: return vmxHCExitTripleFault(pVCpu, pVmxTransient);
9557	case VMX_EXIT_NMI_WINDOW: return vmxHCExitNmiWindow(pVCpu, pVmxTransient);
9558	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return vmxHCExitErrInvalidGuestState(pVCpu, pVmxTransient);
9559
9560	case VMX_EXIT_INIT_SIGNAL:
9561	case VMX_EXIT_SIPI:
9562	case VMX_EXIT_IO_SMI:
9563	case VMX_EXIT_SMI:
9564	case VMX_EXIT_ERR_MSR_LOAD:
9565	case VMX_EXIT_ERR_MACHINE_CHECK:
9566	case VMX_EXIT_PML_FULL:
9567	case VMX_EXIT_VIRTUALIZED_EOI:
9568	case VMX_EXIT_GDTR_IDTR_ACCESS:
9569	case VMX_EXIT_LDTR_TR_ACCESS:
9570	case VMX_EXIT_APIC_WRITE:
9571	case VMX_EXIT_RDRAND:
9572	case VMX_EXIT_RSM:
9573	case VMX_EXIT_VMFUNC:
9574	case VMX_EXIT_ENCLS:
9575	case VMX_EXIT_RDSEED:
9576	case VMX_EXIT_XSAVES:
9577	case VMX_EXIT_XRSTORS:
9578	case VMX_EXIT_UMWAIT:
9579	case VMX_EXIT_TPAUSE:
9580	case VMX_EXIT_LOADIWKEY:
9581	default:
9582	return vmxHCExitErrUnexpected(pVCpu, pVmxTransient);
9583	}
9584	#undef VMEXIT_CALL_RET
9585	}
9586	#endif /* !HMVMX_USE_FUNCTION_TABLE */
9587
9588
9589	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9590	/**
9591	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
9592	*
9593	* @returns Strict VBox status code (i.e. informational status codes too).
9594	* @param pVCpu The cross context virtual CPU structure.
9595	* @param pVmxTransient The VMX-transient structure.
9596	*/
9597	DECLINLINE(VBOXSTRICTRC) vmxHCHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9598	{
9599	uint32_t const uExitReason = pVmxTransient->uExitReason;
9600	switch (uExitReason)
9601	{
9602	case VMX_EXIT_EPT_MISCONFIG: return vmxHCExitEptMisconfig(pVCpu, pVmxTransient);
9603	case VMX_EXIT_EPT_VIOLATION: return vmxHCExitEptViolation(pVCpu, pVmxTransient);
9604	case VMX_EXIT_XCPT_OR_NMI: return vmxHCExitXcptOrNmiNested(pVCpu, pVmxTransient);
9605	case VMX_EXIT_IO_INSTR: return vmxHCExitIoInstrNested(pVCpu, pVmxTransient);
9606	case VMX_EXIT_HLT: return vmxHCExitHltNested(pVCpu, pVmxTransient);
9607
9608	/*
9609	* We shouldn't direct host physical interrupts to the nested-guest.
9610	*/
9611	case VMX_EXIT_EXT_INT:
9612	return vmxHCExitExtInt(pVCpu, pVmxTransient);
9613
9614	/*
9615	* Instructions that cause VM-exits unconditionally or the condition is
9616	* always is taken solely from the nested hypervisor (meaning if the VM-exit
9617	* happens, it's guaranteed to be a nested-guest VM-exit).
9618	*
9619	* - Provides VM-exit instruction length ONLY.
9620	*/
9621	case VMX_EXIT_CPUID: /* Unconditional. */
9622	case VMX_EXIT_VMCALL:
9623	case VMX_EXIT_GETSEC:
9624	case VMX_EXIT_INVD:
9625	case VMX_EXIT_XSETBV:
9626	case VMX_EXIT_VMLAUNCH:
9627	case VMX_EXIT_VMRESUME:
9628	case VMX_EXIT_VMXOFF:
9629	case VMX_EXIT_ENCLS: /* Condition specified solely by nested hypervisor. */
9630	case VMX_EXIT_VMFUNC:
9631	return vmxHCExitInstrNested(pVCpu, pVmxTransient);
9632
9633	/*
9634	* Instructions that cause VM-exits unconditionally or the condition is
9635	* always is taken solely from the nested hypervisor (meaning if the VM-exit
9636	* happens, it's guaranteed to be a nested-guest VM-exit).
9637	*
9638	* - Provides VM-exit instruction length.
9639	* - Provides VM-exit information.
9640	* - Optionally provides Exit qualification.
9641	*
9642	* Since Exit qualification is 0 for all VM-exits where it is not
9643	* applicable, reading and passing it to the guest should produce
9644	* defined behavior.
9645	*
9646	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
9647	*/
9648	case VMX_EXIT_INVEPT: /* Unconditional. */
9649	case VMX_EXIT_INVVPID:
9650	case VMX_EXIT_VMCLEAR:
9651	case VMX_EXIT_VMPTRLD:
9652	case VMX_EXIT_VMPTRST:
9653	case VMX_EXIT_VMXON:
9654	case VMX_EXIT_GDTR_IDTR_ACCESS: /* Condition specified solely by nested hypervisor. */
9655	case VMX_EXIT_LDTR_TR_ACCESS:
9656	case VMX_EXIT_RDRAND:
9657	case VMX_EXIT_RDSEED:
9658	case VMX_EXIT_XSAVES:
9659	case VMX_EXIT_XRSTORS:
9660	case VMX_EXIT_UMWAIT:
9661	case VMX_EXIT_TPAUSE:
9662	return vmxHCExitInstrWithInfoNested(pVCpu, pVmxTransient);
9663
9664	case VMX_EXIT_RDTSC: return vmxHCExitRdtscNested(pVCpu, pVmxTransient);
9665	case VMX_EXIT_RDTSCP: return vmxHCExitRdtscpNested(pVCpu, pVmxTransient);
9666	case VMX_EXIT_RDMSR: return vmxHCExitRdmsrNested(pVCpu, pVmxTransient);
9667	case VMX_EXIT_WRMSR: return vmxHCExitWrmsrNested(pVCpu, pVmxTransient);
9668	case VMX_EXIT_INVLPG: return vmxHCExitInvlpgNested(pVCpu, pVmxTransient);
9669	case VMX_EXIT_INVPCID: return vmxHCExitInvpcidNested(pVCpu, pVmxTransient);
9670	case VMX_EXIT_TASK_SWITCH: return vmxHCExitTaskSwitchNested(pVCpu, pVmxTransient);
9671	case VMX_EXIT_WBINVD: return vmxHCExitWbinvdNested(pVCpu, pVmxTransient);
9672	case VMX_EXIT_MTF: return vmxHCExitMtfNested(pVCpu, pVmxTransient);
9673	case VMX_EXIT_APIC_ACCESS: return vmxHCExitApicAccessNested(pVCpu, pVmxTransient);
9674	case VMX_EXIT_APIC_WRITE: return vmxHCExitApicWriteNested(pVCpu, pVmxTransient);
9675	case VMX_EXIT_VIRTUALIZED_EOI: return vmxHCExitVirtEoiNested(pVCpu, pVmxTransient);
9676	case VMX_EXIT_MOV_CRX: return vmxHCExitMovCRxNested(pVCpu, pVmxTransient);
9677	case VMX_EXIT_INT_WINDOW: return vmxHCExitIntWindowNested(pVCpu, pVmxTransient);
9678	case VMX_EXIT_NMI_WINDOW: return vmxHCExitNmiWindowNested(pVCpu, pVmxTransient);
9679	case VMX_EXIT_TPR_BELOW_THRESHOLD: return vmxHCExitTprBelowThresholdNested(pVCpu, pVmxTransient);
9680	case VMX_EXIT_MWAIT: return vmxHCExitMwaitNested(pVCpu, pVmxTransient);
9681	case VMX_EXIT_MONITOR: return vmxHCExitMonitorNested(pVCpu, pVmxTransient);
9682	case VMX_EXIT_PAUSE: return vmxHCExitPauseNested(pVCpu, pVmxTransient);
9683
9684	case VMX_EXIT_PREEMPT_TIMER:
9685	{
9686	/** @todo NSTVMX: Preempt timer. */
9687	return vmxHCExitPreemptTimer(pVCpu, pVmxTransient);
9688	}
9689
9690	case VMX_EXIT_MOV_DRX: return vmxHCExitMovDRxNested(pVCpu, pVmxTransient);
9691	case VMX_EXIT_RDPMC: return vmxHCExitRdpmcNested(pVCpu, pVmxTransient);
9692
9693	case VMX_EXIT_VMREAD:
9694	case VMX_EXIT_VMWRITE: return vmxHCExitVmreadVmwriteNested(pVCpu, pVmxTransient);
9695
9696	case VMX_EXIT_TRIPLE_FAULT: return vmxHCExitTripleFaultNested(pVCpu, pVmxTransient);
9697	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return vmxHCExitErrInvalidGuestStateNested(pVCpu, pVmxTransient);
9698
9699	case VMX_EXIT_INIT_SIGNAL:
9700	case VMX_EXIT_SIPI:
9701	case VMX_EXIT_IO_SMI:
9702	case VMX_EXIT_SMI:
9703	case VMX_EXIT_ERR_MSR_LOAD:
9704	case VMX_EXIT_ERR_MACHINE_CHECK:
9705	case VMX_EXIT_PML_FULL:
9706	case VMX_EXIT_RSM:
9707	default:
9708	return vmxHCExitErrUnexpected(pVCpu, pVmxTransient);
9709	}
9710	}
9711	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
9712
9713
9714	/** @name VM-exit helpers.
9715	* @{
9716	*/
9717	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9718	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit helpers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9719	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9720
9721	/** Macro for VM-exits called unexpectedly. */
9722	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
9723	do { \
9724	VCPU_2_VMXSTATE((a_pVCpu)).u32HMError = (a_HmError); \
9725	return VERR_VMX_UNEXPECTED_EXIT; \
9726	} while (0)
9727
9728	#ifdef VBOX_STRICT
9729	# ifdef IN_RING0
9730	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
9731	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
9732	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
9733
9734	# define HMVMX_ASSERT_PREEMPT_CPUID() \
9735	do { \
9736	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
9737	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
9738	} while (0)
9739
9740	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
9741	do { \
9742	AssertPtr((a_pVCpu)); \
9743	AssertPtr((a_pVmxTransient)); \
9744	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
9745	Assert((a_pVmxTransient)->pVmcsInfo); \
9746	Assert(ASMIntAreEnabled()); \
9747	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
9748	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
9749	Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
9750	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
9751	if (!VMMRZCallRing3IsEnabled((a_pVCpu))) \
9752	HMVMX_ASSERT_PREEMPT_CPUID(); \
9753	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9754	} while (0)
9755	# else
9756	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() do { } while(0)
9757	# define HMVMX_ASSERT_PREEMPT_CPUID() do { } while(0)
9758	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
9759	do { \
9760	AssertPtr((a_pVCpu)); \
9761	AssertPtr((a_pVmxTransient)); \
9762	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
9763	Assert((a_pVmxTransient)->pVmcsInfo); \
9764	Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
9765	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9766	} while (0)
9767	# endif
9768
9769	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
9770	do { \
9771	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
9772	Assert((a_pVmxTransient)->fIsNestedGuest); \
9773	} while (0)
9774
9775	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
9776	do { \
9777	Log4Func(("\n")); \
9778	} while (0)
9779	#else
9780	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
9781	do { \
9782	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9783	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
9784	} while (0)
9785
9786	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
9787	do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
9788
9789	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
9790	#endif
9791
9792	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9793	/** Macro that does the necessary privilege checks and intercepted VM-exits for
9794	* guests that attempted to execute a VMX instruction. */
9795	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
9796	do \
9797	{ \
9798	VBOXSTRICTRC rcStrictTmp = vmxHCCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
9799	if (rcStrictTmp == VINF_SUCCESS) \
9800	{ /* likely */ } \
9801	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
9802	{ \
9803	Assert((a_pVCpu)->hm.s.Event.fPending); \
9804	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
9805	return VINF_SUCCESS; \
9806	} \
9807	else \
9808	{ \
9809	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
9810	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
9811	} \
9812	} while (0)
9813
9814	/** Macro that decodes a memory operand for an VM-exit caused by an instruction. */
9815	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
9816	do \
9817	{ \
9818	VBOXSTRICTRC rcStrictTmp = vmxHCDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
9819	(a_pGCPtrEffAddr)); \
9820	if (rcStrictTmp == VINF_SUCCESS) \
9821	{ /* likely */ } \
9822	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
9823	{ \
9824	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
9825	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
9826	NOREF(uXcptTmp); \
9827	return VINF_SUCCESS; \
9828	} \
9829	else \
9830	{ \
9831	Log4Func(("vmxHCDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
9832	return rcStrictTmp; \
9833	} \
9834	} while (0)
9835	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
9836
9837
9838	/**
9839	* Advances the guest RIP by the specified number of bytes.
9840	*
9841	* @param pVCpu The cross context virtual CPU structure.
9842	* @param cbInstr Number of bytes to advance the RIP by.
9843	*
9844	* @remarks No-long-jump zone!!!
9845	*/
9846	DECLINLINE(void) vmxHCAdvanceGuestRipBy(PVMCPUCC pVCpu, uint32_t cbInstr)
9847	{
9848	/* Advance the RIP. */
9849	pVCpu->cpum.GstCtx.rip += cbInstr;
9850	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP);
9851
9852	/* Update interrupt inhibition. */
9853	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
9854	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
9855	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
9856	}
9857
9858
9859	/**
9860	* Advances the guest RIP after reading it from the VMCS.
9861	*
9862	* @returns VBox status code, no informational status codes.
9863	* @param pVCpu The cross context virtual CPU structure.
9864	* @param pVmxTransient The VMX-transient structure.
9865	*
9866	* @remarks No-long-jump zone!!!
9867	*/
9868	static int vmxHCAdvanceGuestRip(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9869	{
9870	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9871	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
9872	AssertRCReturn(rc, rc);
9873
9874	vmxHCAdvanceGuestRipBy(pVCpu, pVmxTransient->cbExitInstr);
9875	return VINF_SUCCESS;
9876	}
9877
9878
9879	/**
9880	* Handle a condition that occurred while delivering an event through the guest or
9881	* nested-guest IDT.
9882	*
9883	* @returns Strict VBox status code (i.e. informational status codes too).
9884	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
9885	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
9886	* to continue execution of the guest which will delivery the \#DF.
9887	* @retval VINF_EM_RESET if we detected a triple-fault condition.
9888	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
9889	*
9890	* @param pVCpu The cross context virtual CPU structure.
9891	* @param pVmxTransient The VMX-transient structure.
9892	*
9893	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
9894	* Additionally, HMVMX_READ_EXIT_QUALIFICATION is required if the VM-exit
9895	* is due to an EPT violation, PML full or SPP-related event.
9896	*
9897	* @remarks No-long-jump zone!!!
9898	*/
9899	static VBOXSTRICTRC vmxHCCheckExitDueToEventDelivery(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9900	{
9901	Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending);
9902	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
9903	if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
9904	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
9905	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
9906	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_EXIT_QUALIFICATION);
9907
9908	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
9909	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9910	uint32_t const uIdtVectorInfo = pVmxTransient->uIdtVectoringInfo;
9911	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
9912	if (VMX_IDT_VECTORING_INFO_IS_VALID(uIdtVectorInfo))
9913	{
9914	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(uIdtVectorInfo);
9915	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(uIdtVectorInfo);
9916
9917	/*
9918	* If the event was a software interrupt (generated with INT n) or a software exception
9919	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
9920	* can handle the VM-exit and continue guest execution which will re-execute the
9921	* instruction rather than re-injecting the exception, as that can cause premature
9922	* trips to ring-3 before injection and involve TRPM which currently has no way of
9923	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
9924	* the problem).
9925	*/
9926	IEMXCPTRAISE enmRaise;
9927	IEMXCPTRAISEINFO fRaiseInfo;
9928	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
9929	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
9930	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
9931	{
9932	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
9933	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
9934	}
9935	else if (VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo))
9936	{
9937	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
9938	uint8_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
9939	Assert(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT);
9940
9941	uint32_t const fIdtVectorFlags = vmxHCGetIemXcptFlags(uIdtVector, uIdtVectorType);
9942	uint32_t const fExitVectorFlags = vmxHCGetIemXcptFlags(uExitVector, uExitVectorType);
9943
9944	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
9945
9946	/* Determine a vectoring #PF condition, see comment in vmxHCExitXcptPF(). */
9947	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
9948	{
9949	pVmxTransient->fVectoringPF = true;
9950	enmRaise = IEMXCPTRAISE_PREV_EVENT;
9951	}
9952	}
9953	else
9954	{
9955	/*
9956	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
9957	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
9958	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
9959	*/
9960	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
9961	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
9962	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
9963	enmRaise = IEMXCPTRAISE_PREV_EVENT;
9964	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
9965	}
9966
9967	/*
9968	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
9969	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
9970	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
9971	* subsequent VM-entry would fail, see @bugref{7445}.
9972	*
9973	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception".
9974	*/
9975	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
9976	&& enmRaise == IEMXCPTRAISE_PREV_EVENT
9977	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9978	&& CPUMIsGuestNmiBlocking(pVCpu))
9979	{
9980	CPUMSetGuestNmiBlocking(pVCpu, false);
9981	}
9982
9983	switch (enmRaise)
9984	{
9985	case IEMXCPTRAISE_CURRENT_XCPT:
9986	{
9987	Log4Func(("IDT: Pending secondary Xcpt: idtinfo=%#RX64 exitinfo=%#RX64\n", uIdtVectorInfo, uExitIntInfo));
9988	Assert(rcStrict == VINF_SUCCESS);
9989	break;
9990	}
9991
9992	case IEMXCPTRAISE_PREV_EVENT:
9993	{
9994	uint32_t u32ErrCode;
9995	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(uIdtVectorInfo))
9996	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
9997	else
9998	u32ErrCode = 0;
9999
10000	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see vmxHCExitXcptPF(). */
10001	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectReflect);
10002	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(uIdtVectorInfo), 0 /* cbInstr */,
10003	u32ErrCode, pVCpu->cpum.GstCtx.cr2);
10004
10005	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
10006	VCPU_2_VMXSTATE(pVCpu).Event.u32ErrCode));
10007	Assert(rcStrict == VINF_SUCCESS);
10008	break;
10009	}
10010
10011	case IEMXCPTRAISE_REEXEC_INSTR:
10012	Assert(rcStrict == VINF_SUCCESS);
10013	break;
10014
10015	case IEMXCPTRAISE_DOUBLE_FAULT:
10016	{
10017	/*
10018	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
10019	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
10020	*/
10021	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
10022	{
10023	pVmxTransient->fVectoringDoublePF = true;
10024	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
10025	pVCpu->cpum.GstCtx.cr2));
10026	rcStrict = VINF_SUCCESS;
10027	}
10028	else
10029	{
10030	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectConvertDF);
10031	vmxHCSetPendingXcptDF(pVCpu);
10032	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
10033	uIdtVector, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
10034	rcStrict = VINF_HM_DOUBLE_FAULT;
10035	}
10036	break;
10037	}
10038
10039	case IEMXCPTRAISE_TRIPLE_FAULT:
10040	{
10041	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector,
10042	VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
10043	rcStrict = VINF_EM_RESET;
10044	break;
10045	}
10046
10047	case IEMXCPTRAISE_CPU_HANG:
10048	{
10049	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
10050	rcStrict = VERR_EM_GUEST_CPU_HANG;
10051	break;
10052	}
10053
10054	default:
10055	{
10056	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
10057	rcStrict = VERR_VMX_IPE_2;
10058	break;
10059	}
10060	}
10061	}
10062	else if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
10063	&& !CPUMIsGuestNmiBlocking(pVCpu))
10064	{
10065	if ( VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo)
10066	&& VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo) != X86_XCPT_DF
10067	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(uExitIntInfo))
10068	{
10069	/*
10070	* Execution of IRET caused a fault when NMI blocking was in effect (i.e we're in
10071	* the guest or nested-guest NMI handler). We need to set the block-by-NMI field so
10072	* that virtual NMIs remain blocked until the IRET execution is completed.
10073	*
10074	* See Intel spec. 31.7.1.2 "Resuming Guest Software After Handling An Exception".
10075	*/
10076	CPUMSetGuestNmiBlocking(pVCpu, true);
10077	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
10078	}
10079	else if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
10080	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
10081	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
10082	{
10083	/*
10084	* Execution of IRET caused an EPT violation, page-modification log-full event or
10085	* SPP-related event VM-exit when NMI blocking was in effect (i.e. we're in the
10086	* guest or nested-guest NMI handler). We need to set the block-by-NMI field so
10087	* that virtual NMIs remain blocked until the IRET execution is completed.
10088	*
10089	* See Intel spec. 27.2.3 "Information about NMI unblocking due to IRET"
10090	*/
10091	if (VMX_EXIT_QUAL_EPT_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitQual))
10092	{
10093	CPUMSetGuestNmiBlocking(pVCpu, true);
10094	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
10095	}
10096	}
10097	}
10098
10099	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
10100	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
10101	return rcStrict;
10102	}
10103
10104
10105	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10106	/**
10107	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
10108	* guest attempting to execute a VMX instruction.
10109	*
10110	* @returns Strict VBox status code (i.e. informational status codes too).
10111	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
10112	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
10113	*
10114	* @param pVCpu The cross context virtual CPU structure.
10115	* @param uExitReason The VM-exit reason.
10116	*
10117	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
10118	* @remarks No-long-jump zone!!!
10119	*/
10120	static VBOXSTRICTRC vmxHCCheckExitDueToVmxInstr(PVMCPUCC pVCpu, uint32_t uExitReason)
10121	{
10122	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
10123	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
10124
10125	/*
10126	* The physical CPU would have already checked the CPU mode/code segment.
10127	* We shall just assert here for paranoia.
10128	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
10129	*/
10130	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
10131	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
10132	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
10133
10134	if (uExitReason == VMX_EXIT_VMXON)
10135	{
10136	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
10137
10138	/*
10139	* We check CR4.VMXE because it is required to be always set while in VMX operation
10140	* by physical CPUs and our CR4 read-shadow is only consulted when executing specific
10141	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
10142	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
10143	*/
10144	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
10145	{
10146	Log4Func(("CR4.VMXE is not set -> #UD\n"));
10147	vmxHCSetPendingXcptUD(pVCpu);
10148	return VINF_HM_PENDING_XCPT;
10149	}
10150	}
10151	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
10152	{
10153	/*
10154	* The guest has not entered VMX operation but attempted to execute a VMX instruction
10155	* (other than VMXON), we need to raise a #UD.
10156	*/
10157	Log4Func(("Not in VMX root mode -> #UD\n"));
10158	vmxHCSetPendingXcptUD(pVCpu);
10159	return VINF_HM_PENDING_XCPT;
10160	}
10161
10162	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
10163	return VINF_SUCCESS;
10164	}
10165
10166
10167	/**
10168	* Decodes the memory operand of an instruction that caused a VM-exit.
10169	*
10170	* The Exit qualification field provides the displacement field for memory
10171	* operand instructions, if any.
10172	*
10173	* @returns Strict VBox status code (i.e. informational status codes too).
10174	* @retval VINF_SUCCESS if the operand was successfully decoded.
10175	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
10176	* operand.
10177	* @param pVCpu The cross context virtual CPU structure.
10178	* @param uExitInstrInfo The VM-exit instruction information field.
10179	* @param enmMemAccess The memory operand's access type (read or write).
10180	* @param GCPtrDisp The instruction displacement field, if any. For
10181	* RIP-relative addressing pass RIP + displacement here.
10182	* @param pGCPtrMem Where to store the effective destination memory address.
10183	*
10184	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
10185	* virtual-8086 mode hence skips those checks while verifying if the
10186	* segment is valid.
10187	*/
10188	static VBOXSTRICTRC vmxHCDecodeMemOperand(PVMCPUCC pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
10189	PRTGCPTR pGCPtrMem)
10190	{
10191	Assert(pGCPtrMem);
10192	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
10193	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
10194	\| CPUMCTX_EXTRN_CR0);
10195
10196	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
10197	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
10198	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
10199
10200	VMXEXITINSTRINFO ExitInstrInfo;
10201	ExitInstrInfo.u = uExitInstrInfo;
10202	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
10203	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
10204	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
10205	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
10206	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
10207	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
10208	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
10209	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
10210	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
10211
10212	/*
10213	* Validate instruction information.
10214	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
10215	*/
10216	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
10217	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
10218	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
10219	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
10220	AssertLogRelMsgReturn(fIsMemOperand,
10221	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
10222
10223	/*
10224	* Compute the complete effective address.
10225	*
10226	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
10227	* See AMD spec. 4.5.2 "Segment Registers".
10228	*/
10229	RTGCPTR GCPtrMem = GCPtrDisp;
10230	if (fBaseRegValid)
10231	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
10232	if (fIdxRegValid)
10233	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
10234
10235	RTGCPTR const GCPtrOff = GCPtrMem;
10236	if ( !fIsLongMode
10237	\|\| iSegReg >= X86_SREG_FS)
10238	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
10239	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
10240
10241	/*
10242	* Validate effective address.
10243	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
10244	*/
10245	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
10246	Assert(cbAccess > 0);
10247	if (fIsLongMode)
10248	{
10249	if (X86_IS_CANONICAL(GCPtrMem))
10250	{
10251	*pGCPtrMem = GCPtrMem;
10252	return VINF_SUCCESS;
10253	}
10254
10255	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
10256	* "Data Limit Checks in 64-bit Mode". */
10257	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
10258	vmxHCSetPendingXcptGP(pVCpu, 0);
10259	return VINF_HM_PENDING_XCPT;
10260	}
10261
10262	/*
10263	* This is a watered down version of iemMemApplySegment().
10264	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
10265	* and segment CPL/DPL checks are skipped.
10266	*/
10267	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
10268	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
10269	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
10270
10271	/* Check if the segment is present and usable. */
10272	if ( pSel->Attr.n.u1Present
10273	&& !pSel->Attr.n.u1Unusable)
10274	{
10275	Assert(pSel->Attr.n.u1DescType);
10276	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
10277	{
10278	/* Check permissions for the data segment. */
10279	if ( enmMemAccess == VMXMEMACCESS_WRITE
10280	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
10281	{
10282	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
10283	vmxHCSetPendingXcptGP(pVCpu, iSegReg);
10284	return VINF_HM_PENDING_XCPT;
10285	}
10286
10287	/* Check limits if it's a normal data segment. */
10288	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
10289	{
10290	if ( GCPtrFirst32 > pSel->u32Limit
10291	\|\| GCPtrLast32 > pSel->u32Limit)
10292	{
10293	Log4Func(("Data segment limit exceeded. "
10294	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
10295	GCPtrLast32, pSel->u32Limit));
10296	if (iSegReg == X86_SREG_SS)
10297	vmxHCSetPendingXcptSS(pVCpu, 0);
10298	else
10299	vmxHCSetPendingXcptGP(pVCpu, 0);
10300	return VINF_HM_PENDING_XCPT;
10301	}
10302	}
10303	else
10304	{
10305	/* Check limits if it's an expand-down data segment.
10306	Note! The upper boundary is defined by the B bit, not the G bit! */
10307	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
10308	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
10309	{
10310	Log4Func(("Expand-down data segment limit exceeded. "
10311	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
10312	GCPtrLast32, pSel->u32Limit));
10313	if (iSegReg == X86_SREG_SS)
10314	vmxHCSetPendingXcptSS(pVCpu, 0);
10315	else
10316	vmxHCSetPendingXcptGP(pVCpu, 0);
10317	return VINF_HM_PENDING_XCPT;
10318	}
10319	}
10320	}
10321	else
10322	{
10323	/* Check permissions for the code segment. */
10324	if ( enmMemAccess == VMXMEMACCESS_WRITE
10325	\|\| ( enmMemAccess == VMXMEMACCESS_READ
10326	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
10327	{
10328	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
10329	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
10330	vmxHCSetPendingXcptGP(pVCpu, 0);
10331	return VINF_HM_PENDING_XCPT;
10332	}
10333
10334	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
10335	if ( GCPtrFirst32 > pSel->u32Limit
10336	\|\| GCPtrLast32 > pSel->u32Limit)
10337	{
10338	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
10339	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
10340	if (iSegReg == X86_SREG_SS)
10341	vmxHCSetPendingXcptSS(pVCpu, 0);
10342	else
10343	vmxHCSetPendingXcptGP(pVCpu, 0);
10344	return VINF_HM_PENDING_XCPT;
10345	}
10346	}
10347	}
10348	else
10349	{
10350	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
10351	vmxHCSetPendingXcptGP(pVCpu, 0);
10352	return VINF_HM_PENDING_XCPT;
10353	}
10354
10355	*pGCPtrMem = GCPtrMem;
10356	return VINF_SUCCESS;
10357	}
10358	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10359
10360
10361	/**
10362	* VM-exit helper for LMSW.
10363	*/
10364	static VBOXSTRICTRC vmxHCExitLmsw(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst)
10365	{
10366	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
10367	AssertRCReturn(rc, rc);
10368
10369	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
10370	AssertMsg( rcStrict == VINF_SUCCESS
10371	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10372
10373	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
10374	if (rcStrict == VINF_IEM_RAISED_XCPT)
10375	{
10376	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
10377	rcStrict = VINF_SUCCESS;
10378	}
10379
10380	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitLmsw);
10381	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10382	return rcStrict;
10383	}
10384
10385
10386	/**
10387	* VM-exit helper for CLTS.
10388	*/
10389	static VBOXSTRICTRC vmxHCExitClts(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
10390	{
10391	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
10392	AssertRCReturn(rc, rc);
10393
10394	VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
10395	AssertMsg( rcStrict == VINF_SUCCESS
10396	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10397
10398	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
10399	if (rcStrict == VINF_IEM_RAISED_XCPT)
10400	{
10401	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
10402	rcStrict = VINF_SUCCESS;
10403	}
10404
10405	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitClts);
10406	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10407	return rcStrict;
10408	}
10409
10410
10411	/**
10412	* VM-exit helper for MOV from CRx (CRx read).
10413	*/
10414	static VBOXSTRICTRC vmxHCExitMovFromCrX(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
10415	{
10416	Assert(iCrReg < 16);
10417	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
10418
10419	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
10420	AssertRCReturn(rc, rc);
10421
10422	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
10423	AssertMsg( rcStrict == VINF_SUCCESS
10424	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10425
10426	if (iGReg == X86_GREG_xSP)
10427	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
10428	else
10429	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
10430	#ifdef VBOX_WITH_STATISTICS
10431	switch (iCrReg)
10432	{
10433	case 0: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR0Read); break;
10434	case 2: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR2Read); break;
10435	case 3: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR3Read); break;
10436	case 4: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR4Read); break;
10437	case 8: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR8Read); break;
10438	}
10439	#endif
10440	Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
10441	return rcStrict;
10442	}
10443
10444
10445	/**
10446	* VM-exit helper for MOV to CRx (CRx write).
10447	*/
10448	static VBOXSTRICTRC vmxHCExitMovToCrX(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
10449	{
10450	HMVMX_CPUMCTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
10451
10452	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
10453	AssertMsg( rcStrict == VINF_SUCCESS
10454	\|\| rcStrict == VINF_IEM_RAISED_XCPT
10455	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10456
10457	switch (iCrReg)
10458	{
10459	case 0:
10460	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0
10461	\| HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
10462	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR0Write);
10463	Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
10464	break;
10465
10466	case 2:
10467	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR2Write);
10468	/* Nothing to do here, CR2 it's not part of the VMCS. */
10469	break;
10470
10471	case 3:
10472	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
10473	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR3Write);
10474	Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
10475	break;
10476
10477	case 4:
10478	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
10479	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR4Write);
10480	#ifdef IN_RING0
10481	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
10482	pVCpu->cpum.GstCtx.cr4, pVCpu->hmr0.s.fLoadSaveGuestXcr0));
10483	#else
10484	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr4));
10485	#endif
10486	break;
10487
10488	case 8:
10489	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged,
10490	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
10491	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR8Write);
10492	break;
10493
10494	default:
10495	AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
10496	break;
10497	}
10498
10499	if (rcStrict == VINF_IEM_RAISED_XCPT)
10500	{
10501	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
10502	rcStrict = VINF_SUCCESS;
10503	}
10504	return rcStrict;
10505	}
10506
10507
10508	/**
10509	* VM-exit exception handler for \#PF (Page-fault exception).
10510	*
10511	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
10512	*/
10513	static VBOXSTRICTRC vmxHCExitXcptPF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10514	{
10515	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10516	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
10517
10518	#ifdef IN_RING0
10519	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10520	if (!VM_IS_VMX_NESTED_PAGING(pVM))
10521	{ /* likely */ }
10522	else
10523	#endif
10524	{
10525	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF) && defined(IN_RING0)
10526	Assert(pVmxTransient->fIsNestedGuest \|\| pVCpu->hmr0.s.fUsingDebugLoop);
10527	#endif
10528	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
10529	if (!pVmxTransient->fVectoringDoublePF)
10530	{
10531	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
10532	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
10533	}
10534	else
10535	{
10536	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10537	Assert(!pVmxTransient->fIsNestedGuest);
10538	vmxHCSetPendingXcptDF(pVCpu);
10539	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
10540	}
10541	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestPF);
10542	return VINF_SUCCESS;
10543	}
10544
10545	Assert(!pVmxTransient->fIsNestedGuest);
10546
10547	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
10548	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
10549	if (pVmxTransient->fVectoringPF)
10550	{
10551	Assert(VCPU_2_VMXSTATE(pVCpu).Event.fPending);
10552	return VINF_EM_RAW_INJECT_TRPM_EVENT;
10553	}
10554
10555	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10556	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10557	AssertRCReturn(rc, rc);
10558
10559	Log4Func(("#PF: cs:rip=%#04x:%#RX64 err_code=%#RX32 exit_qual=%#RX64 cr3=%#RX64\n", pCtx->cs.Sel, pCtx->rip,
10560	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual, pCtx->cr3));
10561
10562	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
10563	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
10564
10565	Log4Func(("#PF: rc=%Rrc\n", rc));
10566	if (rc == VINF_SUCCESS)
10567	{
10568	/*
10569	* This is typically a shadow page table sync or a MMIO instruction. But we may have
10570	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
10571	*/
10572	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
10573	TRPMResetTrap(pVCpu);
10574	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitShadowPF);
10575	return rc;
10576	}
10577
10578	if (rc == VINF_EM_RAW_GUEST_TRAP)
10579	{
10580	if (!pVmxTransient->fVectoringDoublePF)
10581	{
10582	/* It's a guest page fault and needs to be reflected to the guest. */
10583	uint32_t const uGstErrorCode = TRPMGetErrorCode(pVCpu);
10584	TRPMResetTrap(pVCpu);
10585	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false; /* In case it's a contributory #PF. */
10586	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
10587	uGstErrorCode, pVmxTransient->uExitQual);
10588	}
10589	else
10590	{
10591	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10592	TRPMResetTrap(pVCpu);
10593	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
10594	vmxHCSetPendingXcptDF(pVCpu);
10595	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
10596	}
10597
10598	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestPF);
10599	return VINF_SUCCESS;
10600	}
10601
10602	TRPMResetTrap(pVCpu);
10603	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitShadowPFEM);
10604	return rc;
10605	}
10606
10607
10608	/**
10609	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
10610	*
10611	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
10612	*/
10613	static VBOXSTRICTRC vmxHCExitXcptMF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10614	{
10615	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10616	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestMF);
10617
10618	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
10619	AssertRCReturn(rc, rc);
10620
10621	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
10622	{
10623	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
10624	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
10625
10626	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
10627	* provides VM-exit instruction length. If this causes problem later,
10628	* disassemble the instruction like it's done on AMD-V. */
10629	int rc2 = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
10630	AssertRCReturn(rc2, rc2);
10631	return rc;
10632	}
10633
10634	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbExitInstr,
10635	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10636	return VINF_SUCCESS;
10637	}
10638
10639
10640	/**
10641	* VM-exit exception handler for \#BP (Breakpoint exception).
10642	*
10643	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
10644	*/
10645	static VBOXSTRICTRC vmxHCExitXcptBP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10646	{
10647	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10648	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestBP);
10649
10650	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10651	AssertRCReturn(rc, rc);
10652
10653	if (!pVmxTransient->fIsNestedGuest)
10654	rc = DBGFTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx));
10655	else
10656	rc = VINF_EM_RAW_GUEST_TRAP;
10657
10658	if (rc == VINF_EM_RAW_GUEST_TRAP)
10659	{
10660	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10661	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10662	rc = VINF_SUCCESS;
10663	}
10664
10665	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_DBG_BREAKPOINT);
10666	return rc;
10667	}
10668
10669
10670	/**
10671	* VM-exit exception handler for \#AC (Alignment-check exception).
10672	*
10673	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
10674	*/
10675	static VBOXSTRICTRC vmxHCExitXcptAC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10676	{
10677	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10678
10679	/*
10680	* Detect #ACs caused by host having enabled split-lock detection.
10681	* Emulate such instructions.
10682	*/
10683	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo,
10684	CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS);
10685	AssertRCReturn(rc, rc);
10686	/** @todo detect split lock in cpu feature? */
10687	if ( /* 1. If 486-style alignment checks aren't enabled, then this must be a split-lock exception */
10688	!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM)
10689	/* 2. #AC cannot happen in rings 0-2 except for split-lock detection. */
10690	\|\| CPUMGetGuestCPL(pVCpu) != 3
10691	/* 3. When the EFLAGS.AC != 0 this can only be a split-lock case. */
10692	\|\| !(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_AC) )
10693	{
10694	/*
10695	* Check for debug/trace events and import state accordingly.
10696	*/
10697	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestACSplitLock);
10698	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10699	if ( !DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_VMX_SPLIT_LOCK)
10700	#ifdef IN_RING0
10701	&& !VBOXVMM_VMX_SPLIT_LOCK_ENABLED()
10702	#endif
10703	)
10704	{
10705	if (pVM->cCpus == 1)
10706	{
10707	#if 0 /** @todo r=bird: This is potentially wrong. Might have to just do a whole state sync above and mark everything changed to be safe... */
10708	rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
10709	#else
10710	rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10711	#endif
10712	AssertRCReturn(rc, rc);
10713	}
10714	}
10715	else
10716	{
10717	rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10718	AssertRCReturn(rc, rc);
10719
10720	VBOXVMM_XCPT_DF(pVCpu, &pVCpu->cpum.GstCtx);
10721
10722	if (DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_VMX_SPLIT_LOCK))
10723	{
10724	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, DBGFEVENT_VMX_SPLIT_LOCK, DBGFEVENTCTX_HM, 0);
10725	if (rcStrict != VINF_SUCCESS)
10726	return rcStrict;
10727	}
10728	}
10729
10730	/*
10731	* Emulate the instruction.
10732	*
10733	* We have to ignore the LOCK prefix here as we must not retrigger the
10734	* detection on the host. This isn't all that satisfactory, though...
10735	*/
10736	if (pVM->cCpus == 1)
10737	{
10738	Log8Func(("cs:rip=%#04x:%#RX64 rflags=%#RX64 cr0=%#RX64 split-lock #AC\n", pVCpu->cpum.GstCtx.cs.Sel,
10739	pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags, pVCpu->cpum.GstCtx.cr0));
10740
10741	/** @todo For SMP configs we should do a rendezvous here. */
10742	VBOXSTRICTRC rcStrict = IEMExecOneIgnoreLock(pVCpu);
10743	if (rcStrict == VINF_SUCCESS)
10744	#if 0 /** @todo r=bird: This is potentially wrong. Might have to just do a whole state sync above and mark everything changed to be safe... */
10745	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged,
10746	HM_CHANGED_GUEST_RIP
10747	\| HM_CHANGED_GUEST_RFLAGS
10748	\| HM_CHANGED_GUEST_GPRS_MASK
10749	\| HM_CHANGED_GUEST_CS
10750	\| HM_CHANGED_GUEST_SS);
10751	#else
10752	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
10753	#endif
10754	else if (rcStrict == VINF_IEM_RAISED_XCPT)
10755	{
10756	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
10757	rcStrict = VINF_SUCCESS;
10758	}
10759	return rcStrict;
10760	}
10761	Log8Func(("cs:rip=%#04x:%#RX64 rflags=%#RX64 cr0=%#RX64 split-lock #AC -> VINF_EM_EMULATE_SPLIT_LOCK\n",
10762	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags, pVCpu->cpum.GstCtx.cr0));
10763	return VINF_EM_EMULATE_SPLIT_LOCK;
10764	}
10765
10766	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestAC);
10767	Log8Func(("cs:rip=%#04x:%#RX64 rflags=%#RX64 cr0=%#RX64 cpl=%d -> #AC\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
10768	pVCpu->cpum.GstCtx.rflags, pVCpu->cpum.GstCtx.cr0, CPUMGetGuestCPL(pVCpu) ));
10769
10770	/* Re-inject it. We'll detect any nesting before getting here. */
10771	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10772	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10773	return VINF_SUCCESS;
10774	}
10775
10776
10777	/**
10778	* VM-exit exception handler for \#DB (Debug exception).
10779	*
10780	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
10781	*/
10782	static VBOXSTRICTRC vmxHCExitXcptDB(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10783	{
10784	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10785	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDB);
10786
10787	/*
10788	* Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
10789	*/
10790	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
10791
10792	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10793	uint64_t const uDR6 = X86_DR6_INIT_VAL
10794	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
10795	\| X86_DR6_BD \| X86_DR6_BS));
10796
10797	int rc;
10798	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10799	if (!pVmxTransient->fIsNestedGuest)
10800	{
10801	rc = DBGFTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, VCPU_2_VMXSTATE(pVCpu).fSingleInstruction);
10802
10803	/*
10804	* Prevents stepping twice over the same instruction when the guest is stepping using
10805	* EFLAGS.TF and the hypervisor debugger is stepping using MTF.
10806	* Testcase: DOSQEMM, break (using "ba x 1") at cs:rip 0x70:0x774 and step (using "t").
10807	*/
10808	if ( rc == VINF_EM_DBG_STEPPED
10809	&& (pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG))
10810	{
10811	Assert(VCPU_2_VMXSTATE(pVCpu).fSingleInstruction);
10812	rc = VINF_EM_RAW_GUEST_TRAP;
10813	}
10814	}
10815	else
10816	rc = VINF_EM_RAW_GUEST_TRAP;
10817	Log6Func(("rc=%Rrc\n", rc));
10818	if (rc == VINF_EM_RAW_GUEST_TRAP)
10819	{
10820	/*
10821	* The exception was for the guest. Update DR6, DR7.GD and
10822	* IA32_DEBUGCTL.LBR before forwarding it.
10823	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
10824	*/
10825	#ifdef IN_RING0
10826	VMMRZCallRing3Disable(pVCpu);
10827	HM_DISABLE_PREEMPT(pVCpu);
10828
10829	pCtx->dr[6] &= ~X86_DR6_B_MASK;
10830	pCtx->dr[6] \|= uDR6;
10831	if (CPUMIsGuestDebugStateActive(pVCpu))
10832	ASMSetDR6(pCtx->dr[6]);
10833
10834	HM_RESTORE_PREEMPT();
10835	VMMRZCallRing3Enable(pVCpu);
10836	#else
10837	/** @todo */
10838	#endif
10839
10840	rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
10841	AssertRCReturn(rc, rc);
10842
10843	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10844	pCtx->dr[7] &= ~(uint64_t)X86_DR7_GD;
10845
10846	/* Paranoia. */
10847	pCtx->dr[7] &= ~(uint64_t)X86_DR7_RAZ_MASK;
10848	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
10849
10850	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
10851	AssertRC(rc);
10852
10853	/*
10854	* Raise #DB in the guest.
10855	*
10856	* It is important to reflect exactly what the VM-exit gave us (preserving the
10857	* interruption-type) rather than use vmxHCSetPendingXcptDB() as the #DB could've
10858	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
10859	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
10860	*
10861	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
10862	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
10863	*/
10864	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10865	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10866	return VINF_SUCCESS;
10867	}
10868
10869	/*
10870	* Not a guest trap, must be a hypervisor related debug event then.
10871	* Update DR6 in case someone is interested in it.
10872	*/
10873	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10874	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
10875	CPUMSetHyperDR6(pVCpu, uDR6);
10876
10877	return rc;
10878	}
10879
10880
10881	/**
10882	* Hacks its way around the lovely mesa driver's backdoor accesses.
10883	*
10884	* @sa hmR0SvmHandleMesaDrvGp.
10885	*/
10886	static int vmxHCHandleMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
10887	{
10888	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
10889	RT_NOREF(pCtx);
10890
10891	/* For now we'll just skip the instruction. */
10892	return vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
10893	}
10894
10895
10896	/**
10897	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
10898	* backdoor logging w/o checking what it is running inside.
10899	*
10900	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
10901	* backdoor port and magic numbers loaded in registers.
10902	*
10903	* @returns true if it is, false if it isn't.
10904	* @sa hmR0SvmIsMesaDrvGp.
10905	*/
10906	DECLINLINE(bool) vmxHCIsMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
10907	{
10908	/* 0xed: IN eAX,dx */
10909	uint8_t abInstr[1];
10910	if (pVmxTransient->cbExitInstr != sizeof(abInstr))
10911	return false;
10912
10913	/* Check that it is #GP(0). */
10914	if (pVmxTransient->uExitIntErrorCode != 0)
10915	return false;
10916
10917	/* Check magic and port. */
10918	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
10919	/Log(("vmxHCIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
10920	if (pCtx->rax != UINT32_C(0x564d5868))
10921	return false;
10922	if (pCtx->dx != UINT32_C(0x5658))
10923	return false;
10924
10925	/* Flat ring-3 CS. */
10926	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
10927	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
10928	/Log(("vmxHCIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
10929	if (pCtx->cs.Attr.n.u2Dpl != 3)
10930	return false;
10931	if (pCtx->cs.u64Base != 0)
10932	return false;
10933
10934	/* Check opcode. */
10935	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
10936	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
10937	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
10938	/Log(("vmxHCIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
10939	if (RT_FAILURE(rc))
10940	return false;
10941	if (abInstr[0] != 0xed)
10942	return false;
10943
10944	return true;
10945	}
10946
10947
10948	/**
10949	* VM-exit exception handler for \#GP (General-protection exception).
10950	*
10951	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
10952	*/
10953	static VBOXSTRICTRC vmxHCExitXcptGP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10954	{
10955	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10956	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestGP);
10957
10958	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10959	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10960	#ifdef IN_RING0
10961	PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
10962	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
10963	{ /* likely */ }
10964	else
10965	#endif
10966	{
10967	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10968	# ifdef IN_RING0
10969	Assert(pVCpu->hmr0.s.fUsingDebugLoop \|\| VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
10970	# else
10971	Assert(/pVCpu->hmr0.s.fUsingDebugLoop \|\|/ VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
10972	# endif
10973	#endif
10974	/*
10975	* If the guest is not in real-mode or we have unrestricted guest execution support, or if we are
10976	* executing a nested-guest, reflect #GP to the guest or nested-guest.
10977	*/
10978	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10979	AssertRCReturn(rc, rc);
10980	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
10981	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
10982
10983	if ( pVmxTransient->fIsNestedGuest
10984	\|\| !VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv
10985	\|\| !vmxHCIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
10986	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10987	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10988	else
10989	rc = vmxHCHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
10990	return rc;
10991	}
10992
10993	#ifdef IN_RING0
10994	Assert(CPUMIsGuestInRealModeEx(pCtx));
10995	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fUnrestrictedGuest);
10996	Assert(!pVmxTransient->fIsNestedGuest);
10997
10998	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10999	AssertRCReturn(rc, rc);
11000
11001	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
11002	if (rcStrict == VINF_SUCCESS)
11003	{
11004	if (!CPUMIsGuestInRealModeEx(pCtx))
11005	{
11006	/*
11007	* The guest is no longer in real-mode, check if we can continue executing the
11008	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
11009	*/
11010	pVmcsInfoShared->RealMode.fRealOnV86Active = false;
11011	if (HMCanExecuteVmxGuest(pVCpu->CTX_SUFF(pVM), pVCpu, pCtx))
11012	{
11013	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
11014	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
11015	}
11016	else
11017	{
11018	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
11019	rcStrict = VINF_EM_RESCHEDULE;
11020	}
11021	}
11022	else
11023	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
11024	}
11025	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11026	{
11027	rcStrict = VINF_SUCCESS;
11028	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11029	}
11030	return VBOXSTRICTRC_VAL(rcStrict);
11031	#endif
11032	}
11033
11034
11035	/**
11036	* VM-exit exception handler wrapper for all other exceptions that are not handled
11037	* by a specific handler.
11038	*
11039	* This simply re-injects the exception back into the VM without any special
11040	* processing.
11041	*
11042	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
11043	*/
11044	static VBOXSTRICTRC vmxHCExitXcptOthers(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11045	{
11046	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11047
11048	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11049	# ifdef IN_RING0
11050	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11051	AssertMsg(pVCpu->hmr0.s.fUsingDebugLoop \|\| pVmcsInfo->pShared->RealMode.fRealOnV86Active \|\| pVmxTransient->fIsNestedGuest,
11052	("uVector=%#x u32XcptBitmap=%#X32\n",
11053	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
11054	NOREF(pVmcsInfo);
11055	# endif
11056	#endif
11057
11058	/*
11059	* Re-inject the exception into the guest. This cannot be a double-fault condition which
11060	* would have been handled while checking exits due to event delivery.
11061	*/
11062	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11063
11064	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11065	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11066	AssertRCReturn(rc, rc);
11067	Log4Func(("Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11068	#endif
11069
11070	#ifdef VBOX_WITH_STATISTICS
11071	switch (uVector)
11072	{
11073	case X86_XCPT_DE: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDE); break;
11074	case X86_XCPT_DB: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDB); break;
11075	case X86_XCPT_BP: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestBP); break;
11076	case X86_XCPT_OF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestOF); break;
11077	case X86_XCPT_BR: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestBR); break;
11078	case X86_XCPT_UD: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestUD); break;
11079	case X86_XCPT_NM: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestOF); break;
11080	case X86_XCPT_DF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDF); break;
11081	case X86_XCPT_TS: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestTS); break;
11082	case X86_XCPT_NP: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestNP); break;
11083	case X86_XCPT_SS: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestSS); break;
11084	case X86_XCPT_GP: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestGP); break;
11085	case X86_XCPT_PF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestPF); break;
11086	case X86_XCPT_MF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestMF); break;
11087	case X86_XCPT_AC: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestAC); break;
11088	case X86_XCPT_XF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestXF); break;
11089	default:
11090	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestXcpUnk);
11091	break;
11092	}
11093	#endif
11094
11095	/* We should never call this function for a page-fault, we'd need to pass on the fault address below otherwise. */
11096	Assert(!VMX_EXIT_INT_INFO_IS_XCPT_PF(pVmxTransient->uExitIntInfo));
11097	NOREF(uVector);
11098
11099	/* Re-inject the original exception into the guest. */
11100	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11101	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11102	return VINF_SUCCESS;
11103	}
11104
11105
11106	/**
11107	* VM-exit exception handler for all exceptions (except NMIs!).
11108	*
11109	* @remarks This may be called for both guests and nested-guests. Take care to not
11110	* make assumptions and avoid doing anything that is not relevant when
11111	* executing a nested-guest (e.g., Mesa driver hacks).
11112	*/
11113	static VBOXSTRICTRC vmxHCExitXcpt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11114	{
11115	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
11116
11117	/*
11118	* If this VM-exit occurred while delivering an event through the guest IDT, take
11119	* action based on the return code and additional hints (e.g. for page-faults)
11120	* that will be updated in the VMX transient structure.
11121	*/
11122	VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
11123	if (rcStrict == VINF_SUCCESS)
11124	{
11125	/*
11126	* If an exception caused a VM-exit due to delivery of an event, the original
11127	* event may have to be re-injected into the guest. We shall reinject it and
11128	* continue guest execution. However, page-fault is a complicated case and
11129	* needs additional processing done in vmxHCExitXcptPF().
11130	*/
11131	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11132	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11133	if ( !VCPU_2_VMXSTATE(pVCpu).Event.fPending
11134	\|\| uVector == X86_XCPT_PF)
11135	{
11136	switch (uVector)
11137	{
11138	case X86_XCPT_PF: return vmxHCExitXcptPF(pVCpu, pVmxTransient);
11139	case X86_XCPT_GP: return vmxHCExitXcptGP(pVCpu, pVmxTransient);
11140	case X86_XCPT_MF: return vmxHCExitXcptMF(pVCpu, pVmxTransient);
11141	case X86_XCPT_DB: return vmxHCExitXcptDB(pVCpu, pVmxTransient);
11142	case X86_XCPT_BP: return vmxHCExitXcptBP(pVCpu, pVmxTransient);
11143	case X86_XCPT_AC: return vmxHCExitXcptAC(pVCpu, pVmxTransient);
11144	default:
11145	return vmxHCExitXcptOthers(pVCpu, pVmxTransient);
11146	}
11147	}
11148	/* else: inject pending event before resuming guest execution. */
11149	}
11150	else if (rcStrict == VINF_HM_DOUBLE_FAULT)
11151	{
11152	Assert(VCPU_2_VMXSTATE(pVCpu).Event.fPending);
11153	rcStrict = VINF_SUCCESS;
11154	}
11155
11156	return rcStrict;
11157	}
11158	/** @} */
11159
11160
11161	/** @name VM-exit handlers.
11162	* @{
11163	*/
11164	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11165	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11166	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11167
11168	/**
11169	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11170	*/
11171	HMVMX_EXIT_DECL vmxHCExitExtInt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11172	{
11173	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11174	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitExtInt);
11175
11176	#ifdef IN_RING0
11177	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11178	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11179	return VINF_SUCCESS;
11180	return VINF_EM_RAW_INTERRUPT;
11181	#else
11182	return VINF_SUCCESS;
11183	#endif
11184	}
11185
11186
11187	/**
11188	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
11189	* VM-exit.
11190	*/
11191	HMVMX_EXIT_DECL vmxHCExitXcptOrNmi(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11192	{
11193	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11194	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitXcptNmi, y3);
11195
11196	vmxHCReadExitIntInfoVmcs(pVCpu, pVmxTransient);
11197
11198	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11199	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11200	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11201
11202	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11203	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
11204	&& uExitIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
11205	NOREF(pVmcsInfo);
11206
11207	VBOXSTRICTRC rcStrict;
11208	switch (uExitIntType)
11209	{
11210	#ifdef IN_RING0 /* NMIs should never reach R3. */
11211	/*
11212	* Host physical NMIs:
11213	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
11214	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
11215	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
11216	*
11217	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11218	* See Intel spec. 27.5.5 "Updating Non-Register State".
11219	*/
11220	case VMX_EXIT_INT_INFO_TYPE_NMI:
11221	{
11222	rcStrict = vmxHCExitHostNmi(pVCpu, pVmcsInfo);
11223	break;
11224	}
11225	#endif
11226
11227	/*
11228	* Privileged software exceptions (#DB from ICEBP),
11229	* Software exceptions (#BP and #OF),
11230	* Hardware exceptions:
11231	* Process the required exceptions and resume guest execution if possible.
11232	*/
11233	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11234	Assert(uVector == X86_XCPT_DB);
11235	RT_FALL_THRU();
11236	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11237	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uExitIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
11238	RT_FALL_THRU();
11239	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11240	{
11241	NOREF(uVector);
11242	vmxHCReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
11243	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
11244	vmxHCReadIdtVectoringInfoVmcs(pVCpu, pVmxTransient);
11245	vmxHCReadIdtVectoringErrorCodeVmcs(pVCpu, pVmxTransient);
11246
11247	rcStrict = vmxHCExitXcpt(pVCpu, pVmxTransient);
11248	break;
11249	}
11250
11251	default:
11252	{
11253	VCPU_2_VMXSTATE(pVCpu).u32HMError = pVmxTransient->uExitIntInfo;
11254	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11255	AssertMsgFailed(("Invalid/unexpected VM-exit interruption info %#x\n", pVmxTransient->uExitIntInfo));
11256	break;
11257	}
11258	}
11259
11260	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitXcptNmi, y3);
11261	return rcStrict;
11262	}
11263
11264
11265	/**
11266	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11267	*/
11268	HMVMX_EXIT_NSRC_DECL vmxHCExitIntWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11269	{
11270	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11271
11272	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11273	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11274	vmxHCClearIntWindowExitVmcs(pVCpu, pVmcsInfo);
11275
11276	/* Evaluate and deliver pending events and resume guest execution. */
11277	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitIntWindow);
11278	return VINF_SUCCESS;
11279	}
11280
11281
11282	/**
11283	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11284	*/
11285	HMVMX_EXIT_NSRC_DECL vmxHCExitNmiWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11286	{
11287	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11288
11289	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11290	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
11291	{
11292	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11293	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
11294	}
11295
11296	Assert(!CPUMIsGuestNmiBlocking(pVCpu));
11297
11298	/*
11299	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11300	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11301	*/
11302	uint32_t fIntrState;
11303	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11304	AssertRC(rc);
11305	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
11306	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
11307	{
11308	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11309	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11310
11311	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
11312	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, fIntrState);
11313	AssertRC(rc);
11314	}
11315
11316	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11317	vmxHCClearNmiWindowExitVmcs(pVCpu, pVmcsInfo);
11318
11319	/* Evaluate and deliver pending events and resume guest execution. */
11320	return VINF_SUCCESS;
11321	}
11322
11323
11324	/**
11325	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11326	*/
11327	HMVMX_EXIT_NSRC_DECL vmxHCExitWbinvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11328	{
11329	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11330	return vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
11331	}
11332
11333
11334	/**
11335	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11336	*/
11337	HMVMX_EXIT_NSRC_DECL vmxHCExitInvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11338	{
11339	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11340	return vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
11341	}
11342
11343
11344	/**
11345	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11346	*/
11347	HMVMX_EXIT_DECL vmxHCExitCpuid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11348	{
11349	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11350
11351	/*
11352	* Get the state we need and update the exit history entry.
11353	*/
11354	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11355	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
11356
11357	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
11358	AssertRCReturn(rc, rc);
11359
11360	VBOXSTRICTRC rcStrict;
11361	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11362	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
11363	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11364	if (!pExitRec)
11365	{
11366	/*
11367	* Regular CPUID instruction execution.
11368	*/
11369	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbExitInstr);
11370	if (rcStrict == VINF_SUCCESS)
11371	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11372	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11373	{
11374	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11375	rcStrict = VINF_SUCCESS;
11376	}
11377	}
11378	else
11379	{
11380	/*
11381	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
11382	*/
11383	int rc2 = vmxHCImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11384	AssertRCReturn(rc2, rc2);
11385
11386	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11387	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11388
11389	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11390	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
11391
11392	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11393	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11394	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11395	}
11396	return rcStrict;
11397	}
11398
11399
11400	/**
11401	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11402	*/
11403	HMVMX_EXIT_DECL vmxHCExitGetsec(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11404	{
11405	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11406
11407	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11408	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
11409	AssertRCReturn(rc, rc);
11410
11411	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
11412	return VINF_EM_RAW_EMULATE_INSTR;
11413
11414	AssertMsgFailed(("vmxHCExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
11415	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
11416	}
11417
11418
11419	/**
11420	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11421	*/
11422	HMVMX_EXIT_DECL vmxHCExitRdtsc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11423	{
11424	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11425
11426	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11427	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
11428	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
11429	AssertRCReturn(rc, rc);
11430
11431	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbExitInstr);
11432	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11433	{
11434	/* If we get a spurious VM-exit when TSC offsetting is enabled,
11435	we must reset offsetting on VM-entry. See @bugref{6634}. */
11436	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11437	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11438	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11439	}
11440	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11441	{
11442	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11443	rcStrict = VINF_SUCCESS;
11444	}
11445	return rcStrict;
11446	}
11447
11448
11449	/**
11450	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11451	*/
11452	HMVMX_EXIT_DECL vmxHCExitRdtscp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11453	{
11454	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11455
11456	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11457	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
11458	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
11459	AssertRCReturn(rc, rc);
11460
11461	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbExitInstr);
11462	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11463	{
11464	/* If we get a spurious VM-exit when TSC offsetting is enabled,
11465	we must reset offsetting on VM-reentry. See @bugref{6634}. */
11466	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11467	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11468	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11469	}
11470	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11471	{
11472	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11473	rcStrict = VINF_SUCCESS;
11474	}
11475	return rcStrict;
11476	}
11477
11478
11479	/**
11480	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11481	*/
11482	HMVMX_EXIT_DECL vmxHCExitRdpmc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11483	{
11484	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11485
11486	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11487	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
11488	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
11489	AssertRCReturn(rc, rc);
11490
11491	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11492	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
11493	if (RT_LIKELY(rc == VINF_SUCCESS))
11494	{
11495	rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
11496	Assert(pVmxTransient->cbExitInstr == 2);
11497	}
11498	else
11499	{
11500	AssertMsgFailed(("vmxHCExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11501	rc = VERR_EM_INTERPRETER;
11502	}
11503	return rc;
11504	}
11505
11506
11507	/**
11508	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11509	*/
11510	HMVMX_EXIT_DECL vmxHCExitVmcall(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11511	{
11512	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11513
11514	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11515	if (EMAreHypercallInstructionsEnabled(pVCpu))
11516	{
11517	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11518	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
11519	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
11520	AssertRCReturn(rc, rc);
11521
11522	/* Perform the hypercall. */
11523	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
11524	if (rcStrict == VINF_SUCCESS)
11525	{
11526	rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
11527	AssertRCReturn(rc, rc);
11528	}
11529	else
11530	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11531	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11532	\|\| RT_FAILURE(rcStrict));
11533
11534	/* If the hypercall changes anything other than guest's general-purpose registers,
11535	we would need to reload the guest changed bits here before VM-entry. */
11536	}
11537	else
11538	Log4Func(("Hypercalls not enabled\n"));
11539
11540	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11541	if (RT_FAILURE(rcStrict))
11542	{
11543	vmxHCSetPendingXcptUD(pVCpu);
11544	rcStrict = VINF_SUCCESS;
11545	}
11546
11547	return rcStrict;
11548	}
11549
11550
11551	/**
11552	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11553	*/
11554	HMVMX_EXIT_DECL vmxHCExitInvlpg(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11555	{
11556	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11557	#ifdef IN_RING0
11558	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging \|\| pVCpu->hmr0.s.fUsingDebugLoop);
11559	#endif
11560
11561	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11562	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
11563	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
11564	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
11565	AssertRCReturn(rc, rc);
11566
11567	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->uExitQual);
11568
11569	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
11570	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11571	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11572	{
11573	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11574	rcStrict = VINF_SUCCESS;
11575	}
11576	else
11577	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
11578	VBOXSTRICTRC_VAL(rcStrict)));
11579	return rcStrict;
11580	}
11581
11582
11583	/**
11584	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11585	*/
11586	HMVMX_EXIT_DECL vmxHCExitMonitor(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11587	{
11588	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11589
11590	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11591	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
11592	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
11593	AssertRCReturn(rc, rc);
11594
11595	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbExitInstr);
11596	if (rcStrict == VINF_SUCCESS)
11597	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11598	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11599	{
11600	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11601	rcStrict = VINF_SUCCESS;
11602	}
11603
11604	return rcStrict;
11605	}
11606
11607
11608	/**
11609	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11610	*/
11611	HMVMX_EXIT_DECL vmxHCExitMwait(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11612	{
11613	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11614
11615	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11616	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
11617	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
11618	AssertRCReturn(rc, rc);
11619
11620	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbExitInstr);
11621	if (RT_SUCCESS(rcStrict))
11622	{
11623	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11624	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
11625	rcStrict = VINF_SUCCESS;
11626	}
11627
11628	return rcStrict;
11629	}
11630
11631
11632	/**
11633	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11634	* VM-exit.
11635	*/
11636	HMVMX_EXIT_DECL vmxHCExitTripleFault(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11637	{
11638	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11639	return VINF_EM_RESET;
11640	}
11641
11642
11643	/**
11644	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11645	*/
11646	HMVMX_EXIT_DECL vmxHCExitHlt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11647	{
11648	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11649
11650	int rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
11651	AssertRCReturn(rc, rc);
11652
11653	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
11654	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
11655	rc = VINF_SUCCESS;
11656	else
11657	rc = VINF_EM_HALT;
11658
11659	if (rc != VINF_SUCCESS)
11660	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchHltToR3);
11661	return rc;
11662	}
11663
11664
11665	/**
11666	* VM-exit handler for instructions that result in a \#UD exception delivered to
11667	* the guest.
11668	*/
11669	HMVMX_EXIT_NSRC_DECL vmxHCExitSetPendingXcptUD(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11670	{
11671	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11672	vmxHCSetPendingXcptUD(pVCpu);
11673	return VINF_SUCCESS;
11674	}
11675
11676
11677	/**
11678	* VM-exit handler for expiry of the VMX-preemption timer.
11679	*/
11680	HMVMX_EXIT_DECL vmxHCExitPreemptTimer(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11681	{
11682	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11683
11684	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
11685	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11686	Log12(("vmxHCExitPreemptTimer:\n"));
11687
11688	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11689	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
11690	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11691	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitPreemptTimer);
11692	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11693	}
11694
11695
11696	/**
11697	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11698	*/
11699	HMVMX_EXIT_DECL vmxHCExitXsetbv(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11700	{
11701	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11702
11703	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11704	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
11705	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
11706	AssertRCReturn(rc, rc);
11707
11708	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbExitInstr);
11709	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
11710	: HM_CHANGED_RAISED_XCPT_MASK);
11711
11712	#ifdef IN_RING0
11713	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11714	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
11715	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
11716	{
11717	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
11718	vmxHCUpdateStartVmFunction(pVCpu);
11719	}
11720	#endif
11721
11722	return rcStrict;
11723	}
11724
11725
11726	/**
11727	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11728	*/
11729	HMVMX_EXIT_DECL vmxHCExitInvpcid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11730	{
11731	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11732
11733	/** @todo Enable the new code after finding a reliably guest test-case. */
11734	#if 1
11735	return VERR_EM_INTERPRETER;
11736	#else
11737	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
11738	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
11739	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
11740	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
11741	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
11742	AssertRCReturn(rc, rc);
11743
11744	/* Paranoia. Ensure this has a memory operand. */
11745	Assert(!pVmxTransient->ExitInstrInfo.Inv.u1Cleared0);
11746
11747	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
11748	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
11749	uint64_t const uType = CPUMIsGuestIn64BitCode(pVCpu) ? pVCpu->cpum.GstCtx.aGRegs[iGReg].u64
11750	: pVCpu->cpum.GstCtx.aGRegs[iGReg].u32;
11751
11752	RTGCPTR GCPtrDesc;
11753	HMVMX_DECODE_MEM_OPERAND(pVCpu, pVmxTransient->ExitInstrInfo.u, pVmxTransient->uExitQual, VMXMEMACCESS_READ, &GCPtrDesc);
11754
11755	VBOXSTRICTRC rcStrict = IEMExecDecodedInvpcid(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->ExitInstrInfo.Inv.iSegReg,
11756	GCPtrDesc, uType);
11757	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11758	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11759	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11760	{
11761	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11762	rcStrict = VINF_SUCCESS;
11763	}
11764	return rcStrict;
11765	#endif
11766	}
11767
11768
11769	/**
11770	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
11771	* VM-exit.
11772	*/
11773	HMVMX_EXIT_NSRC_DECL vmxHCExitErrInvalidGuestState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11774	{
11775	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11776	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11777	AssertRCReturn(rc, rc);
11778
11779	rc = vmxHCCheckCachedVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
11780	if (RT_FAILURE(rc))
11781	return rc;
11782
11783	uint32_t const uInvalidReason = vmxHCCheckGuestState(pVCpu, pVmcsInfo);
11784	NOREF(uInvalidReason);
11785
11786	#ifdef VBOX_STRICT
11787	uint32_t fIntrState;
11788	uint64_t u64Val;
11789	vmxHCReadEntryIntInfoVmcs(pVCpu, pVmxTransient);
11790	vmxHCReadEntryXcptErrorCodeVmcs(pVCpu, pVmxTransient);
11791	vmxHCReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
11792
11793	Log4(("uInvalidReason %u\n", uInvalidReason));
11794	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11795	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11796	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11797
11798	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &fIntrState); AssertRC(rc);
11799	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
11800	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
11801	Log4(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
11802	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_MASK, &u64Val); AssertRC(rc);
11803	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RX64\n", u64Val));
11804	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Val); AssertRC(rc);
11805	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RX64\n", u64Val));
11806	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_MASK, &u64Val); AssertRC(rc);
11807	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RX64\n", u64Val));
11808	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Val); AssertRC(rc);
11809	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RX64\n", u64Val));
11810	# ifdef IN_RING0
11811	if (pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
11812	{
11813	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11814	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11815	}
11816
11817	hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
11818	# endif
11819	#endif
11820
11821	return VERR_VMX_INVALID_GUEST_STATE;
11822	}
11823
11824	/**
11825	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
11826	*/
11827	HMVMX_EXIT_NSRC_DECL vmxHCExitErrUnexpected(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11828	{
11829	/*
11830	* Cumulative notes of all recognized but unexpected VM-exits.
11831	*
11832	* 1. This does -not- cover scenarios like a page-fault VM-exit occurring when
11833	* nested-paging is used.
11834	*
11835	* 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
11836	* emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
11837	* this function (and thereby stop VM execution) for handling such instructions.
11838	*
11839	*
11840	* VMX_EXIT_INIT_SIGNAL:
11841	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11842	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
11843	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
11844	*
11845	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
11846	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
11847	* See Intel spec. "23.8 Restrictions on VMX operation".
11848	*
11849	* VMX_EXIT_SIPI:
11850	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
11851	* activity state is used. We don't make use of it as our guests don't have direct
11852	* access to the host local APIC.
11853	*
11854	* See Intel spec. 25.3 "Other Causes of VM-exits".
11855	*
11856	* VMX_EXIT_IO_SMI:
11857	* VMX_EXIT_SMI:
11858	* This can only happen if we support dual-monitor treatment of SMI, which can be
11859	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
11860	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
11861	* VMX root mode or receive an SMI. If we get here, something funny is going on.
11862	*
11863	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11864	* See Intel spec. 25.3 "Other Causes of VM-Exits"
11865	*
11866	* VMX_EXIT_ERR_MSR_LOAD:
11867	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
11868	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
11869	* execution.
11870	*
11871	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
11872	*
11873	* VMX_EXIT_ERR_MACHINE_CHECK:
11874	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
11875	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
11876	* #MC exception abort class exception is raised. We thus cannot assume a
11877	* reasonable chance of continuing any sort of execution and we bail.
11878	*
11879	* See Intel spec. 15.1 "Machine-check Architecture".
11880	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
11881	*
11882	* VMX_EXIT_PML_FULL:
11883	* VMX_EXIT_VIRTUALIZED_EOI:
11884	* VMX_EXIT_APIC_WRITE:
11885	* We do not currently support any of these features and thus they are all unexpected
11886	* VM-exits.
11887	*
11888	* VMX_EXIT_GDTR_IDTR_ACCESS:
11889	* VMX_EXIT_LDTR_TR_ACCESS:
11890	* VMX_EXIT_RDRAND:
11891	* VMX_EXIT_RSM:
11892	* VMX_EXIT_VMFUNC:
11893	* VMX_EXIT_ENCLS:
11894	* VMX_EXIT_RDSEED:
11895	* VMX_EXIT_XSAVES:
11896	* VMX_EXIT_XRSTORS:
11897	* VMX_EXIT_UMWAIT:
11898	* VMX_EXIT_TPAUSE:
11899	* VMX_EXIT_LOADIWKEY:
11900	* These VM-exits are -not- caused unconditionally by execution of the corresponding
11901	* instruction. Any VM-exit for these instructions indicate a hardware problem,
11902	* unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
11903	*
11904	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
11905	*/
11906	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11907	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
11908	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
11909	}
11910
11911
11912	/**
11913	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11914	*/
11915	HMVMX_EXIT_DECL vmxHCExitRdmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11916	{
11917	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11918
11919	/** @todo Optimize this: We currently drag in the whole MSR state
11920	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11921	* MSRs required. That would require changes to IEM and possibly CPUM too.
11922	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11923	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11924	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
11925	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
11926	switch (idMsr)
11927	{
11928	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
11929	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
11930	}
11931
11932	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
11933	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, fImport);
11934	AssertRCReturn(rc, rc);
11935
11936	Log4Func(("ecx=%#RX32\n", idMsr));
11937
11938	#if defined(VBOX_STRICT) && defined(IN_RING0)
11939	Assert(!pVmxTransient->fIsNestedGuest);
11940	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
11941	{
11942	if ( vmxHCIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
11943	&& idMsr != MSR_K6_EFER)
11944	{
11945	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
11946	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
11947	}
11948	if (vmxHCIsLazyGuestMsr(pVCpu, idMsr))
11949	{
11950	Assert(pVmcsInfo->pvMsrBitmap);
11951	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
11952	if (fMsrpm & VMXMSRPM_ALLOW_RD)
11953	{
11954	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
11955	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
11956	}
11957	}
11958	}
11959	#endif
11960
11961	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbExitInstr);
11962	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitRdmsr);
11963	if (rcStrict == VINF_SUCCESS)
11964	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11965	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11966	{
11967	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11968	rcStrict = VINF_SUCCESS;
11969	}
11970	else
11971	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ \|\| rcStrict == VINF_EM_TRIPLE_FAULT,
11972	("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
11973
11974	return rcStrict;
11975	}
11976
11977
11978	/**
11979	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11980	*/
11981	HMVMX_EXIT_DECL vmxHCExitWrmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11982	{
11983	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11984
11985	/** @todo Optimize this: We currently drag in the whole MSR state
11986	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11987	* MSRs required. That would require changes to IEM and possibly CPUM too.
11988	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11989	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
11990	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
11991
11992	/*
11993	* The FS and GS base MSRs are not part of the above all-MSRs mask.
11994	* Although we don't need to fetch the base as it will be overwritten shortly, while
11995	* loading guest-state we would also load the entire segment register including limit
11996	* and attributes and thus we need to load them here.
11997	*/
11998	switch (idMsr)
11999	{
12000	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
12001	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
12002	}
12003
12004	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12005	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
12006	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, fImport);
12007	AssertRCReturn(rc, rc);
12008
12009	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
12010
12011	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbExitInstr);
12012	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitWrmsr);
12013
12014	if (rcStrict == VINF_SUCCESS)
12015	{
12016	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
12017
12018	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
12019	if ( idMsr == MSR_IA32_APICBASE
12020	\|\| ( idMsr >= MSR_IA32_X2APIC_START
12021	&& idMsr <= MSR_IA32_X2APIC_END))
12022	{
12023	/*
12024	* We've already saved the APIC related guest-state (TPR) in post-run phase.
12025	* When full APIC register virtualization is implemented we'll have to make
12026	* sure APIC state is saved from the VMCS before IEM changes it.
12027	*/
12028	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
12029	}
12030	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12031	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
12032	else if (idMsr == MSR_K6_EFER)
12033	{
12034	/*
12035	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
12036	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
12037	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
12038	*/
12039	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
12040	}
12041
12042	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not used. */
12043	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
12044	{
12045	switch (idMsr)
12046	{
12047	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12048	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12049	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12050	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_FS); break;
12051	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_GS); break;
12052	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
12053	default:
12054	{
12055	#ifdef IN_RING0
12056	if (vmxHCIsLazyGuestMsr(pVCpu, idMsr))
12057	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
12058	else if (vmxHCIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
12059	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12060	#else
12061	AssertMsgFailed(("TODO\n"));
12062	#endif
12063	break;
12064	}
12065	}
12066	}
12067	#if defined(VBOX_STRICT) && defined(IN_RING0)
12068	else
12069	{
12070	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12071	switch (idMsr)
12072	{
12073	case MSR_IA32_SYSENTER_CS:
12074	case MSR_IA32_SYSENTER_EIP:
12075	case MSR_IA32_SYSENTER_ESP:
12076	case MSR_K8_FS_BASE:
12077	case MSR_K8_GS_BASE:
12078	{
12079	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
12080	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
12081	}
12082
12083	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12084	default:
12085	{
12086	if (vmxHCIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
12087	{
12088	/* EFER MSR writes are always intercepted. */
12089	if (idMsr != MSR_K6_EFER)
12090	{
12091	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12092	idMsr));
12093	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
12094	}
12095	}
12096
12097	if (vmxHCIsLazyGuestMsr(pVCpu, idMsr))
12098	{
12099	Assert(pVmcsInfo->pvMsrBitmap);
12100	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
12101	if (fMsrpm & VMXMSRPM_ALLOW_WR)
12102	{
12103	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
12104	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
12105	}
12106	}
12107	break;
12108	}
12109	}
12110	}
12111	#endif /* VBOX_STRICT */
12112	}
12113	else if (rcStrict == VINF_IEM_RAISED_XCPT)
12114	{
12115	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12116	rcStrict = VINF_SUCCESS;
12117	}
12118	else
12119	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE \|\| rcStrict == VINF_EM_TRIPLE_FAULT,
12120	("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
12121
12122	return rcStrict;
12123	}
12124
12125
12126	/**
12127	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12128	*/
12129	HMVMX_EXIT_DECL vmxHCExitPause(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12130	{
12131	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12132
12133	/** @todo The guest has likely hit a contended spinlock. We might want to
12134	* poke a schedule different guest VCPU. */
12135	int rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
12136	if (RT_SUCCESS(rc))
12137	return VINF_EM_RAW_INTERRUPT;
12138
12139	AssertMsgFailed(("vmxHCExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
12140	return rc;
12141	}
12142
12143
12144	/**
12145	* VM-exit handler for when the TPR value is lowered below the specified
12146	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12147	*/
12148	HMVMX_EXIT_NSRC_DECL vmxHCExitTprBelowThreshold(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12149	{
12150	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12151	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
12152
12153	/*
12154	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
12155	* We'll re-evaluate pending interrupts and inject them before the next VM
12156	* entry so we can just continue execution here.
12157	*/
12158	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitTprBelowThreshold);
12159	return VINF_SUCCESS;
12160	}
12161
12162
12163	/**
12164	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12165	* VM-exit.
12166	*
12167	* @retval VINF_SUCCESS when guest execution can continue.
12168	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12169	* @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
12170	* incompatible guest state for VMX execution (real-on-v86 case).
12171	*/
12172	HMVMX_EXIT_DECL vmxHCExitMovCRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12173	{
12174	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12175	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitMovCRx, y2);
12176
12177	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12178	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
12179	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
12180
12181	VBOXSTRICTRC rcStrict;
12182	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
12183	uint64_t const uExitQual = pVmxTransient->uExitQual;
12184	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
12185	switch (uAccessType)
12186	{
12187	/*
12188	* MOV to CRx.
12189	*/
12190	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
12191	{
12192	/*
12193	* When PAE paging is used, the CPU will reload PAE PDPTEs from CR3 when the guest
12194	* changes certain bits even in CR0, CR4 (and not just CR3). We are currently fine
12195	* since IEM_CPUMCTX_EXTRN_MUST_MASK (used below) includes CR3 which will import
12196	* PAE PDPTEs as well.
12197	*/
12198	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
12199	AssertRCReturn(rc, rc);
12200
12201	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
12202	#ifdef IN_RING0
12203	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
12204	#endif
12205	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
12206	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
12207
12208	/*
12209	* MOV to CR3 only cause a VM-exit when one or more of the following are true:
12210	* - When nested paging isn't used.
12211	* - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
12212	* - We are executing in the VM debug loop.
12213	*/
12214	#ifdef IN_RING0
12215	Assert( iCrReg != 3
12216	\|\| !VM_IS_VMX_NESTED_PAGING(pVM)
12217	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
12218	\|\| pVCpu->hmr0.s.fUsingDebugLoop);
12219	#else
12220	Assert( iCrReg != 3
12221	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx));
12222	#endif
12223
12224	/* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
12225	Assert( iCrReg != 8
12226	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
12227
12228	rcStrict = vmxHCExitMovToCrX(pVCpu, pVmxTransient->cbExitInstr, iGReg, iCrReg);
12229	AssertMsg( rcStrict == VINF_SUCCESS
12230	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12231
12232	#ifdef IN_RING0
12233	/*
12234	* This is a kludge for handling switches back to real mode when we try to use
12235	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
12236	* deal with special selector values, so we have to return to ring-3 and run
12237	* there till the selector values are V86 mode compatible.
12238	*
12239	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
12240	* latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
12241	* this function.
12242	*/
12243	if ( iCrReg == 0
12244	&& rcStrict == VINF_SUCCESS
12245	&& !VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
12246	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
12247	&& (uOldCr0 & X86_CR0_PE)
12248	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
12249	{
12250	/** @todo Check selectors rather than returning all the time. */
12251	Assert(!pVmxTransient->fIsNestedGuest);
12252	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
12253	rcStrict = VINF_EM_RESCHEDULE_REM;
12254	}
12255	#endif
12256
12257	break;
12258	}
12259
12260	/*
12261	* MOV from CRx.
12262	*/
12263	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
12264	{
12265	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
12266	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
12267
12268	/*
12269	* MOV from CR3 only cause a VM-exit when one or more of the following are true:
12270	* - When nested paging isn't used.
12271	* - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
12272	* - We are executing in the VM debug loop.
12273	*/
12274	#ifdef IN_RING0
12275	Assert( iCrReg != 3
12276	\|\| !VM_IS_VMX_NESTED_PAGING(pVM)
12277	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
12278	\|\| pVCpu->hmr0.s.fLeaveDone);
12279	#else
12280	Assert( iCrReg != 3
12281	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx));
12282	#endif
12283
12284	/* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12285	Assert( iCrReg != 8
12286	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
12287
12288	rcStrict = vmxHCExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
12289	break;
12290	}
12291
12292	/*
12293	* CLTS (Clear Task-Switch Flag in CR0).
12294	*/
12295	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
12296	{
12297	rcStrict = vmxHCExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr);
12298	break;
12299	}
12300
12301	/*
12302	* LMSW (Load Machine-Status Word into CR0).
12303	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
12304	*/
12305	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
12306	{
12307	RTGCPTR GCPtrEffDst;
12308	uint8_t const cbInstr = pVmxTransient->cbExitInstr;
12309	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
12310	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
12311	if (fMemOperand)
12312	{
12313	vmxHCReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
12314	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
12315	}
12316	else
12317	GCPtrEffDst = NIL_RTGCPTR;
12318	rcStrict = vmxHCExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
12319	break;
12320	}
12321
12322	default:
12323	{
12324	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
12325	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
12326	}
12327	}
12328
12329	Assert((VCPU_2_VMXSTATE(pVCpu).fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
12330	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
12331	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
12332
12333	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitMovCRx, y2);
12334	NOREF(pVM);
12335	return rcStrict;
12336	}
12337
12338
12339	/**
12340	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12341	* VM-exit.
12342	*/
12343	HMVMX_EXIT_DECL vmxHCExitIoInstr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12344	{
12345	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12346	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitIO, y1);
12347
12348	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12349	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12350	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
12351	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
12352	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
12353	\| CPUMCTX_EXTRN_EFER);
12354	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12355	AssertRCReturn(rc, rc);
12356
12357	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12358	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
12359	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
12360	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12361	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
12362	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12363	bool const fDbgStepping = VCPU_2_VMXSTATE(pVCpu).fSingleInstruction;
12364	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
12365
12366	/*
12367	* Update exit history to see if this exit can be optimized.
12368	*/
12369	VBOXSTRICTRC rcStrict;
12370	PCEMEXITREC pExitRec = NULL;
12371	if ( !fGstStepping
12372	&& !fDbgStepping)
12373	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12374	!fIOString
12375	? !fIOWrite
12376	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12377	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12378	: !fIOWrite
12379	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12380	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12381	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12382	if (!pExitRec)
12383	{
12384	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
12385	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12386
12387	uint32_t const cbValue = s_aIOSizes[uIOSize];
12388	uint32_t const cbInstr = pVmxTransient->cbExitInstr;
12389	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12390	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
12391	if (fIOString)
12392	{
12393	/*
12394	* INS/OUTS - I/O String instruction.
12395	*
12396	* Use instruction-information if available, otherwise fall back on
12397	* interpreting the instruction.
12398	*/
12399	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12400	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
12401	bool const fInsOutsInfo = RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
12402	if (fInsOutsInfo)
12403	{
12404	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
12405	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12406	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12407	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12408	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
12409	if (fIOWrite)
12410	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12411	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
12412	else
12413	{
12414	/*
12415	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12416	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12417	* See Intel Instruction spec. for "INS".
12418	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12419	*/
12420	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
12421	}
12422	}
12423	else
12424	rcStrict = IEMExecOne(pVCpu);
12425
12426	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP);
12427	fUpdateRipAlready = true;
12428	}
12429	else
12430	{
12431	/*
12432	* IN/OUT - I/O instruction.
12433	*/
12434	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12435	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
12436	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
12437	if (fIOWrite)
12438	{
12439	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
12440	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitIOWrite);
12441	#ifdef IN_RING0
12442	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12443	&& !pCtx->eflags.Bits.u1TF)
12444	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
12445	#endif
12446	}
12447	else
12448	{
12449	uint32_t u32Result = 0;
12450	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12451	if (IOM_SUCCESS(rcStrict))
12452	{
12453	/* Save result of I/O IN instr. in AL/AX/EAX. */
12454	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
12455	}
12456	#ifdef IN_RING0
12457	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12458	&& !pCtx->eflags.Bits.u1TF)
12459	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
12460	#endif
12461	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitIORead);
12462	}
12463	}
12464
12465	if (IOM_SUCCESS(rcStrict))
12466	{
12467	if (!fUpdateRipAlready)
12468	{
12469	vmxHCAdvanceGuestRipBy(pVCpu, cbInstr);
12470	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP);
12471	}
12472
12473	/*
12474	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12475	* while booting Fedora 17 64-bit guest.
12476	*
12477	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12478	*/
12479	if (fIOString)
12480	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12481
12482	/*
12483	* If any I/O breakpoints are armed, we need to check if one triggered
12484	* and take appropriate action.
12485	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12486	*/
12487	rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
12488	AssertRCReturn(rc, rc);
12489
12490	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12491	* execution engines about whether hyper BPs and such are pending. */
12492	uint32_t const uDr7 = pCtx->dr[7];
12493	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12494	&& X86_DR7_ANY_RW_IO(uDr7)
12495	&& (pCtx->cr4 & X86_CR4_DE))
12496	\|\| DBGFBpIsHwIoArmed(pVM)))
12497	{
12498	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatDRxIoCheck);
12499
12500	#ifdef IN_RING0
12501	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12502	VMMRZCallRing3Disable(pVCpu);
12503	HM_DISABLE_PREEMPT(pVCpu);
12504
12505	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12506
12507	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
12508	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12509	{
12510	/* Raise #DB. */
12511	if (fIsGuestDbgActive)
12512	ASMSetDR6(pCtx->dr[6]);
12513	if (pCtx->dr[7] != uDr7)
12514	VCPU_2_VMXSTATE(pVCpu).fCtxChanged \|= HM_CHANGED_GUEST_DR7;
12515
12516	vmxHCSetPendingXcptDB(pVCpu);
12517	}
12518	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12519	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12520	else if ( rcStrict2 != VINF_SUCCESS
12521	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
12522	rcStrict = rcStrict2;
12523	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12524
12525	HM_RESTORE_PREEMPT();
12526	VMMRZCallRing3Enable(pVCpu);
12527	#else
12528	/** @todo */
12529	#endif
12530	}
12531	}
12532
12533	#ifdef VBOX_STRICT
12534	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12535	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
12536	Assert(!fIOWrite);
12537	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12538	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
12539	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
12540	Assert(fIOWrite);
12541	else
12542	{
12543	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12544	* statuses, that the VMM device and some others may return. See
12545	* IOM_SUCCESS() for guidance. */
12546	AssertMsg( RT_FAILURE(rcStrict)
12547	\|\| rcStrict == VINF_SUCCESS
12548	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
12549	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
12550	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
12551	\|\| rcStrict == VINF_EM_RAW_TO_R3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12552	# endif
12553	}
12554	#endif
12555	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitIO, y1);
12556	}
12557	else
12558	{
12559	/*
12560	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
12561	*/
12562	int rc2 = vmxHCImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12563	AssertRCReturn(rc2, rc2);
12564	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &VCPU_2_VMXSTATS(pVCpu).StatExitIOWrite : &VCPU_2_VMXSTATS(pVCpu).StatExitIORead
12565	: fIOWrite ? &VCPU_2_VMXSTATS(pVCpu).StatExitIOStringWrite : &VCPU_2_VMXSTATS(pVCpu).StatExitIOStringRead);
12566	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12567	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12568	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
12569	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
12570
12571	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12572	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
12573
12574	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12575	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12576	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12577	}
12578	return rcStrict;
12579	}
12580
12581
12582	/**
12583	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12584	* VM-exit.
12585	*/
12586	HMVMX_EXIT_DECL vmxHCExitTaskSwitch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12587	{
12588	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12589
12590	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
12591	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
12592	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12593	{
12594	vmxHCReadIdtVectoringInfoVmcs(pVCpu, pVmxTransient);
12595	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12596	{
12597	uint32_t uErrCode;
12598	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
12599	{
12600	vmxHCReadIdtVectoringErrorCodeVmcs(pVCpu, pVmxTransient);
12601	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12602	}
12603	else
12604	uErrCode = 0;
12605
12606	RTGCUINTPTR GCPtrFaultAddress;
12607	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(pVmxTransient->uIdtVectoringInfo))
12608	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
12609	else
12610	GCPtrFaultAddress = 0;
12611
12612	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
12613
12614	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12615	pVmxTransient->cbExitInstr, uErrCode, GCPtrFaultAddress);
12616
12617	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo),
12618	VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo)));
12619	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitTaskSwitch);
12620	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12621	}
12622	}
12623
12624	/* Fall back to the interpreter to emulate the task-switch. */
12625	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitTaskSwitch);
12626	return VERR_EM_INTERPRETER;
12627	}
12628
12629
12630	/**
12631	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12632	*/
12633	HMVMX_EXIT_DECL vmxHCExitMtf(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12634	{
12635	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12636
12637	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12638	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
12639	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
12640	AssertRC(rc);
12641	return VINF_EM_DBG_STEPPED;
12642	}
12643
12644
12645	/**
12646	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12647	*/
12648	HMVMX_EXIT_DECL vmxHCExitApicAccess(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12649	{
12650	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12651	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitApicAccess);
12652
12653	vmxHCReadExitIntInfoVmcs(pVCpu, pVmxTransient);
12654	vmxHCReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
12655	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
12656	vmxHCReadIdtVectoringInfoVmcs(pVCpu, pVmxTransient);
12657	vmxHCReadIdtVectoringErrorCodeVmcs(pVCpu, pVmxTransient);
12658
12659	/*
12660	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
12661	*/
12662	VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12663	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
12664	{
12665	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12666	if (RT_UNLIKELY(VCPU_2_VMXSTATE(pVCpu).Event.fPending))
12667	{
12668	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectInterpret);
12669	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12670	}
12671	}
12672	else
12673	{
12674	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
12675	return rcStrict;
12676	}
12677
12678	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12679	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12680	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
12681	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
12682	AssertRCReturn(rc, rc);
12683
12684	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12685	uint32_t const uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
12686	switch (uAccessType)
12687	{
12688	#ifdef IN_RING0
12689	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12690	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12691	{
12692	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
12693	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
12694	("vmxHCExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12695
12696	RTGCPHYS GCPhys = VCPU_2_VMXSTATE(pVCpu).vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
12697	GCPhys &= PAGE_BASE_GC_MASK;
12698	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
12699	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12700	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
12701
12702	rcStrict = IOMR0MmioPhysHandler(pVCpu->CTX_SUFF(pVM), pVCpu,
12703	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW, GCPhys);
12704	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12705	if ( rcStrict == VINF_SUCCESS
12706	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12707	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
12708	{
12709	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
12710	\| HM_CHANGED_GUEST_APIC_TPR);
12711	rcStrict = VINF_SUCCESS;
12712	}
12713	break;
12714	}
12715	#else
12716	/** @todo */
12717	#endif
12718
12719	default:
12720	{
12721	Log4Func(("uAccessType=%#x\n", uAccessType));
12722	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12723	break;
12724	}
12725	}
12726
12727	if (rcStrict != VINF_SUCCESS)
12728	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchApicAccessToR3);
12729	return rcStrict;
12730	}
12731
12732
12733	/**
12734	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12735	* VM-exit.
12736	*/
12737	HMVMX_EXIT_DECL vmxHCExitMovDRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12738	{
12739	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12740	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12741
12742	/*
12743	* We might also get this VM-exit if the nested-guest isn't intercepting MOV DRx accesses.
12744	* In such a case, rather than disabling MOV DRx intercepts and resuming execution, we
12745	* must emulate the MOV DRx access.
12746	*/
12747	if (!pVmxTransient->fIsNestedGuest)
12748	{
12749	/* We should -not- get this VM-exit if the guest's debug registers were active. */
12750	if (pVmxTransient->fWasGuestDebugStateActive)
12751	{
12752	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
12753	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
12754	}
12755
12756	if ( !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction
12757	&& !pVmxTransient->fWasHyperDebugStateActive)
12758	{
12759	Assert(!DBGFIsStepping(pVCpu));
12760	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
12761
12762	/* Don't intercept MOV DRx any more. */
12763	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
12764	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
12765	AssertRC(rc);
12766
12767	#ifdef IN_RING0
12768	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12769	VMMRZCallRing3Disable(pVCpu);
12770	HM_DISABLE_PREEMPT(pVCpu);
12771
12772	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12773	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12774	Assert(CPUMIsGuestDebugStateActive(pVCpu));
12775
12776	HM_RESTORE_PREEMPT();
12777	VMMRZCallRing3Enable(pVCpu);
12778	#else
12779	/** @todo */
12780	#endif
12781
12782	#ifdef VBOX_WITH_STATISTICS
12783	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
12784	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12785	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxWrite);
12786	else
12787	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxRead);
12788	#endif
12789	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatDRxContextSwitch);
12790	return VINF_SUCCESS;
12791	}
12792	}
12793
12794	/*
12795	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
12796	* The EFER MSR is always up-to-date.
12797	* Update the segment registers and DR7 from the CPU.
12798	*/
12799	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12800	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
12801	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
12802	AssertRCReturn(rc, rc);
12803	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
12804
12805	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
12806	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12807	{
12808	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12809	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
12810	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
12811	if (RT_SUCCESS(rc))
12812	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_DR7);
12813	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxWrite);
12814	}
12815	else
12816	{
12817	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12818	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
12819	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
12820	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxRead);
12821	}
12822
12823	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
12824	if (RT_SUCCESS(rc))
12825	{
12826	int rc2 = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
12827	AssertRCReturn(rc2, rc2);
12828	return VINF_SUCCESS;
12829	}
12830	return rc;
12831	}
12832
12833
12834	/**
12835	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12836	* Conditional VM-exit.
12837	*/
12838	HMVMX_EXIT_DECL vmxHCExitEptMisconfig(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12839	{
12840	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12841
12842	#ifdef IN_RING0
12843	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
12844
12845	vmxHCReadExitIntInfoVmcs(pVCpu, pVmxTransient);
12846	vmxHCReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
12847	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
12848	vmxHCReadIdtVectoringInfoVmcs(pVCpu, pVmxTransient);
12849	vmxHCReadIdtVectoringErrorCodeVmcs(pVCpu, pVmxTransient);
12850
12851	/*
12852	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
12853	*/
12854	VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12855	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
12856	{
12857	/*
12858	* In the unlikely case where delivering an event causes an EPT misconfig (MMIO), go back to
12859	* instruction emulation to inject the original event. Otherwise, injecting the original event
12860	* using hardware-assisted VMX would trigger the same EPT misconfig VM-exit again.
12861	*/
12862	if (!VCPU_2_VMXSTATE(pVCpu).Event.fPending)
12863	{ /* likely */ }
12864	else
12865	{
12866	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectInterpret);
12867	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12868	/** @todo NSTVMX: Think about how this should be handled. */
12869	if (pVmxTransient->fIsNestedGuest)
12870	return VERR_VMX_IPE_3;
12871	#endif
12872	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12873	}
12874	}
12875	else
12876	{
12877	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
12878	return rcStrict;
12879	}
12880
12881	/*
12882	* Get sufficient state and update the exit history entry.
12883	*/
12884	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12885	vmxHCReadGuestPhysicalAddrVmcs(pVCpu, pVmxTransient);
12886	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
12887	AssertRCReturn(rc, rc);
12888
12889	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
12890	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12891	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
12892	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12893	if (!pExitRec)
12894	{
12895	/*
12896	* If we succeed, resume guest execution.
12897	* If we fail in interpreting the instruction because we couldn't get the guest physical address
12898	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12899	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12900	* weird case. See @bugref{6043}.
12901	*/
12902	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
12903	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12904	/** @todo bird: We can probably just go straight to IOM here and assume that
12905	* it's MMIO, then fall back on PGM if that hunch didn't work out so
12906	* well. However, we need to address that aliasing workarounds that
12907	* PGMR0Trap0eHandlerNPMisconfig implements. So, some care is needed.
12908	*
12909	* Might also be interesting to see if we can get this done more or
12910	* less locklessly inside IOM. Need to consider the lookup table
12911	* updating and use a bit more carefully first (or do all updates via
12912	* rendezvous) */
12913	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
12914	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12915	if ( rcStrict == VINF_SUCCESS
12916	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12917	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
12918	{
12919	/* Successfully handled MMIO operation. */
12920	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
12921	\| HM_CHANGED_GUEST_APIC_TPR);
12922	rcStrict = VINF_SUCCESS;
12923	}
12924	}
12925	else
12926	{
12927	/*
12928	* Frequent exit or something needing probing. Call EMHistoryExec.
12929	*/
12930	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12931	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12932
12933	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12934	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
12935
12936	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12937	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12938	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12939	}
12940	return rcStrict;
12941	#else
12942	AssertFailed();
12943	return VERR_VMX_IPE_3; /* Should never happen with Apple HV in R3. */
12944	#endif
12945	}
12946
12947
12948	/**
12949	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12950	* VM-exit.
12951	*/
12952	HMVMX_EXIT_DECL vmxHCExitEptViolation(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12953	{
12954	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12955	#ifdef IN_RING0
12956	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
12957
12958	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
12959	vmxHCReadExitIntInfoVmcs(pVCpu, pVmxTransient);
12960	vmxHCReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
12961	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
12962	vmxHCReadIdtVectoringInfoVmcs(pVCpu, pVmxTransient);
12963	vmxHCReadIdtVectoringErrorCodeVmcs(pVCpu, pVmxTransient);
12964
12965	/*
12966	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
12967	*/
12968	VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12969	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
12970	{
12971	/*
12972	* If delivery of an event causes an EPT violation (true nested #PF and not MMIO),
12973	* we shall resolve the nested #PF and re-inject the original event.
12974	*/
12975	if (VCPU_2_VMXSTATE(pVCpu).Event.fPending)
12976	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectReflectNPF);
12977	}
12978	else
12979	{
12980	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
12981	return rcStrict;
12982	}
12983
12984	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12985	vmxHCReadGuestPhysicalAddrVmcs(pVCpu, pVmxTransient);
12986	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
12987	AssertRCReturn(rc, rc);
12988
12989	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
12990	uint64_t const uExitQual = pVmxTransient->uExitQual;
12991	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", uExitQual));
12992
12993	RTGCUINT uErrorCode = 0;
12994	if (uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_INSTR_FETCH)
12995	uErrorCode \|= X86_TRAP_PF_ID;
12996	if (uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE)
12997	uErrorCode \|= X86_TRAP_PF_RW;
12998	if (uExitQual & (VMX_EXIT_QUAL_EPT_ENTRY_READ \| VMX_EXIT_QUAL_EPT_ENTRY_WRITE \| VMX_EXIT_QUAL_EPT_ENTRY_EXECUTE))
12999	uErrorCode \|= X86_TRAP_PF_P;
13000
13001	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13002	Log4Func(("at %#RX64 (%#RX64 errcode=%#x) cs:rip=%#04x:%#RX64\n", GCPhys, uExitQual, uErrorCode, pCtx->cs.Sel, pCtx->rip));
13003
13004	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
13005
13006	/*
13007	* Handle the pagefault trap for the nested shadow table.
13008	*/
13009	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
13010	rcStrict = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
13011	TRPMResetTrap(pVCpu);
13012
13013	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
13014	if ( rcStrict == VINF_SUCCESS
13015	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
13016	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
13017	{
13018	/* Successfully synced our nested page tables. */
13019	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitReasonNpf);
13020	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
13021	return VINF_SUCCESS;
13022	}
13023	#else
13024	PVM pVM = pVCpu->CTX_SUFF(pVM);
13025	uint64_t const uHostTsc = ASMReadTSC(); RT_NOREF(uHostTsc);
13026	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13027	vmxHCReadGuestPhysicalAddrVmcs(pVCpu, pVmxTransient);
13028	vmxHCImportGuestRip(pVCpu);
13029	vmxHCImportGuestSegReg(pVCpu, X86_SREG_CS);
13030
13031	/*
13032	* Ask PGM for information about the given GCPhys. We need to check if we're
13033	* out of sync first.
13034	*/
13035	NEMHCDARWINHMACPCCSTATE State = { RT_BOOL(pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE), false, false };
13036	PGMPHYSNEMPAGEINFO Info;
13037	int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, pVmxTransient->uGuestPhysicalAddr, State.fWriteAccess, &Info,
13038	nemR3DarwinHandleMemoryAccessPageCheckerCallback, &State);
13039	if (RT_SUCCESS(rc))
13040	{
13041	if (Info.fNemProt & ( RT_BOOL(pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE)
13042	? NEM_PAGE_PROT_WRITE : NEM_PAGE_PROT_READ))
13043	{
13044	if (State.fCanResume)
13045	{
13046	Log4(("MemExit/%u: %04x:%08RX64: %RGp (=>%RHp) %s fProt=%u%s%s%s; restarting\n",
13047	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13048	pVmxTransient->uGuestPhysicalAddr, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
13049	Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
13050	State.fDidSomething ? "" : " no-change"));
13051	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_MEMORY_ACCESS),
13052	pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, uHostTsc);
13053	return VINF_SUCCESS;
13054	}
13055	}
13056
13057	Log4(("MemExit/%u: %04x:%08RX64: %RGp (=>%RHp) %s fProt=%u%s%s%s; emulating\n",
13058	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13059	pVmxTransient->uGuestPhysicalAddr, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
13060	Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
13061	State.fDidSomething ? "" : " no-change"));
13062	}
13063	else
13064	Log4(("MemExit/%u: %04x:%08RX64: %RGp rc=%Rrc%s; emulating\n",
13065	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13066	pVmxTransient->uGuestPhysicalAddr, rc, State.fDidSomething ? " modified-backing" : ""));
13067
13068	/*
13069	* Emulate the memory access, either access handler or special memory.
13070	*/
13071	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
13072	RT_BOOL(pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE)
13073	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
13074	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
13075	pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, uHostTsc);
13076
13077	rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13078	AssertRCReturn(rc, rc);
13079
13080	VBOXSTRICTRC rcStrict;
13081	if (!pExitRec)
13082	{
13083	rcStrict = IEMExecOne(pVCpu);
13084	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
13085	}
13086	else
13087	{
13088	/* Frequent access or probing. */
13089	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
13090	Log4(("MemExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
13091	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13092	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
13093	}
13094	#endif
13095
13096	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13097	return rcStrict;
13098	}
13099
13100
13101	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13102	/**
13103	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
13104	*/
13105	HMVMX_EXIT_DECL vmxHCExitVmclear(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13106	{
13107	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13108
13109	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13110	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
13111	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13112	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
13113	\| CPUMCTX_EXTRN_HWVIRT
13114	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13115	AssertRCReturn(rc, rc);
13116
13117	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13118
13119	VMXVEXITINFO ExitInfo;
13120	RT_ZERO(ExitInfo);
13121	ExitInfo.uReason = pVmxTransient->uExitReason;
13122	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13123	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13124	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13125	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13126
13127	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
13128	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13129	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
13130	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13131	{
13132	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13133	rcStrict = VINF_SUCCESS;
13134	}
13135	return rcStrict;
13136	}
13137
13138
13139	/**
13140	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
13141	*/
13142	HMVMX_EXIT_DECL vmxHCExitVmlaunch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13143	{
13144	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13145
13146	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
13147	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
13148	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13149	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13150	AssertRCReturn(rc, rc);
13151
13152	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13153
13154	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
13155	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMLAUNCH);
13156	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
13157	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13158	{
13159	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
13160	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
13161	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
13162	}
13163	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
13164	return rcStrict;
13165	}
13166
13167
13168	/**
13169	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
13170	*/
13171	HMVMX_EXIT_DECL vmxHCExitVmptrld(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13172	{
13173	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13174
13175	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13176	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
13177	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13178	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
13179	\| CPUMCTX_EXTRN_HWVIRT
13180	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13181	AssertRCReturn(rc, rc);
13182
13183	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13184
13185	VMXVEXITINFO ExitInfo;
13186	RT_ZERO(ExitInfo);
13187	ExitInfo.uReason = pVmxTransient->uExitReason;
13188	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13189	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13190	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13191	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13192
13193	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
13194	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13195	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
13196	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13197	{
13198	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13199	rcStrict = VINF_SUCCESS;
13200	}
13201	return rcStrict;
13202	}
13203
13204
13205	/**
13206	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
13207	*/
13208	HMVMX_EXIT_DECL vmxHCExitVmptrst(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13209	{
13210	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13211
13212	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13213	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
13214	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13215	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
13216	\| CPUMCTX_EXTRN_HWVIRT
13217	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13218	AssertRCReturn(rc, rc);
13219
13220	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13221
13222	VMXVEXITINFO ExitInfo;
13223	RT_ZERO(ExitInfo);
13224	ExitInfo.uReason = pVmxTransient->uExitReason;
13225	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13226	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13227	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13228	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
13229
13230	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
13231	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13232	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13233	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13234	{
13235	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13236	rcStrict = VINF_SUCCESS;
13237	}
13238	return rcStrict;
13239	}
13240
13241
13242	/**
13243	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
13244	*/
13245	HMVMX_EXIT_DECL vmxHCExitVmread(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13246	{
13247	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13248
13249	/*
13250	* Strictly speaking we should not get VMREAD VM-exits for shadow VMCS fields and
13251	* thus might not need to import the shadow VMCS state, it's safer just in case
13252	* code elsewhere dares look at unsynced VMCS fields.
13253	*/
13254	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13255	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
13256	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13257	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
13258	\| CPUMCTX_EXTRN_HWVIRT
13259	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13260	AssertRCReturn(rc, rc);
13261
13262	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13263
13264	VMXVEXITINFO ExitInfo;
13265	RT_ZERO(ExitInfo);
13266	ExitInfo.uReason = pVmxTransient->uExitReason;
13267	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13268	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13269	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13270	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
13271	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
13272
13273	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
13274	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13275	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13276	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13277	{
13278	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13279	rcStrict = VINF_SUCCESS;
13280	}
13281	return rcStrict;
13282	}
13283
13284
13285	/**
13286	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
13287	*/
13288	HMVMX_EXIT_DECL vmxHCExitVmresume(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13289	{
13290	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13291
13292	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
13293	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
13294	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13295	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13296	AssertRCReturn(rc, rc);
13297
13298	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13299
13300	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
13301	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMRESUME);
13302	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
13303	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13304	{
13305	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
13306	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
13307	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
13308	}
13309	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
13310	return rcStrict;
13311	}
13312
13313
13314	/**
13315	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
13316	*/
13317	HMVMX_EXIT_DECL vmxHCExitVmwrite(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13318	{
13319	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13320
13321	/*
13322	* Although we should not get VMWRITE VM-exits for shadow VMCS fields, since our HM hook
13323	* gets invoked when IEM's VMWRITE instruction emulation modifies the current VMCS and it
13324	* flags re-loading the entire shadow VMCS, we should save the entire shadow VMCS here.
13325	*/
13326	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13327	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
13328	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13329	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
13330	\| CPUMCTX_EXTRN_HWVIRT
13331	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13332	AssertRCReturn(rc, rc);
13333
13334	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13335
13336	VMXVEXITINFO ExitInfo;
13337	RT_ZERO(ExitInfo);
13338	ExitInfo.uReason = pVmxTransient->uExitReason;
13339	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13340	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13341	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13342	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
13343	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13344
13345	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
13346	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13347	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
13348	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13349	{
13350	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13351	rcStrict = VINF_SUCCESS;
13352	}
13353	return rcStrict;
13354	}
13355
13356
13357	/**
13358	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
13359	*/
13360	HMVMX_EXIT_DECL vmxHCExitVmxoff(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13361	{
13362	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13363
13364	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13365	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
13366	\| CPUMCTX_EXTRN_HWVIRT
13367	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13368	AssertRCReturn(rc, rc);
13369
13370	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13371
13372	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbExitInstr);
13373	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13374	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
13375	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13376	{
13377	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13378	rcStrict = VINF_SUCCESS;
13379	}
13380	return rcStrict;
13381	}
13382
13383
13384	/**
13385	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
13386	*/
13387	HMVMX_EXIT_DECL vmxHCExitVmxon(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13388	{
13389	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13390
13391	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13392	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
13393	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13394	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
13395	\| CPUMCTX_EXTRN_HWVIRT
13396	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13397	AssertRCReturn(rc, rc);
13398
13399	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13400
13401	VMXVEXITINFO ExitInfo;
13402	RT_ZERO(ExitInfo);
13403	ExitInfo.uReason = pVmxTransient->uExitReason;
13404	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13405	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13406	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13407	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13408
13409	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
13410	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13411	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
13412	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13413	{
13414	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13415	rcStrict = VINF_SUCCESS;
13416	}
13417	return rcStrict;
13418	}
13419
13420
13421	/**
13422	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
13423	*/
13424	HMVMX_EXIT_DECL vmxHCExitInvvpid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13425	{
13426	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13427
13428	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13429	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
13430	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13431	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
13432	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13433	AssertRCReturn(rc, rc);
13434
13435	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13436
13437	VMXVEXITINFO ExitInfo;
13438	RT_ZERO(ExitInfo);
13439	ExitInfo.uReason = pVmxTransient->uExitReason;
13440	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13441	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13442	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13443	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13444
13445	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
13446	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13447	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13448	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13449	{
13450	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13451	rcStrict = VINF_SUCCESS;
13452	}
13453	return rcStrict;
13454	}
13455	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13456	/** @} */
13457
13458
13459	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13460	/** @name Nested-guest VM-exit handlers.
13461	* @{
13462	*/
13463	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13464	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13465	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13466
13467	/**
13468	* Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
13469	* Conditional VM-exit.
13470	*/
13471	HMVMX_EXIT_DECL vmxHCExitXcptOrNmiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13472	{
13473	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13474
13475	vmxHCReadExitIntInfoVmcs(pVCpu, pVmxTransient);
13476
13477	uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
13478	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
13479	Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));
13480
13481	switch (uExitIntType)
13482	{
13483	/*
13484	* Physical NMIs:
13485	* We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the host.
13486	*/
13487	case VMX_EXIT_INT_INFO_TYPE_NMI:
13488	return vmxHCExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
13489
13490	/*
13491	* Hardware exceptions,
13492	* Software exceptions,
13493	* Privileged software exceptions:
13494	* Figure out if the exception must be delivered to the guest or the nested-guest.
13495	*/
13496	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
13497	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
13498	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
13499	{
13500	vmxHCReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
13501	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13502	vmxHCReadIdtVectoringInfoVmcs(pVCpu, pVmxTransient);
13503	vmxHCReadIdtVectoringErrorCodeVmcs(pVCpu, pVmxTransient);
13504
13505	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13506	bool const fIntercept = CPUMIsGuestVmxXcptInterceptSet(pCtx, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo),
13507	pVmxTransient->uExitIntErrorCode);
13508	if (fIntercept)
13509	{
13510	/* Exit qualification is required for debug and page-fault exceptions. */
13511	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13512
13513	/*
13514	* For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
13515	* software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
13516	* length. However, if delivery of a software interrupt, software exception or privileged
13517	* software exception causes a VM-exit, that too provides the VM-exit instruction length.
13518	*/
13519	VMXVEXITINFO ExitInfo;
13520	RT_ZERO(ExitInfo);
13521	ExitInfo.uReason = pVmxTransient->uExitReason;
13522	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13523	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13524
13525	VMXVEXITEVENTINFO ExitEventInfo;
13526	RT_ZERO(ExitEventInfo);
13527	ExitEventInfo.uExitIntInfo = pVmxTransient->uExitIntInfo;
13528	ExitEventInfo.uExitIntErrCode = pVmxTransient->uExitIntErrorCode;
13529	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
13530	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
13531
13532	#ifdef DEBUG_ramshankar
13533	vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13534	Log4Func(("exit_int_info=%#RX32 err_code=%#RX32 exit_qual=%#RX64\n", pVmxTransient->uExitIntInfo,
13535	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual));
13536	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
13537	{
13538	Log4Func(("idt_info=%#RX32 idt_errcode=%#RX32 cr2=%#RX64\n", pVmxTransient->uIdtVectoringInfo,
13539	pVmxTransient->uIdtVectoringErrorCode, pCtx->cr2));
13540	}
13541	#endif
13542	return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
13543	}
13544
13545	/* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs in vmxHCExitXcptPF. */
13546	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
13547	return vmxHCExitXcpt(pVCpu, pVmxTransient);
13548	}
13549
13550	/*
13551	* Software interrupts:
13552	* VM-exits cannot be caused by software interrupts.
13553	*
13554	* External interrupts:
13555	* This should only happen when "acknowledge external interrupts on VM-exit"
13556	* control is set. However, we never set this when executing a guest or
13557	* nested-guest. For nested-guests it is emulated while injecting interrupts into
13558	* the guest.
13559	*/
13560	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
13561	case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
13562	default:
13563	{
13564	VCPU_2_VMXSTATE(pVCpu).u32HMError = pVmxTransient->uExitIntInfo;
13565	return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
13566	}
13567	}
13568	}
13569
13570
13571	/**
13572	* Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
13573	* Unconditional VM-exit.
13574	*/
13575	HMVMX_EXIT_DECL vmxHCExitTripleFaultNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13576	{
13577	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13578	return IEMExecVmxVmexitTripleFault(pVCpu);
13579	}
13580
13581
13582	/**
13583	* Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
13584	*/
13585	HMVMX_EXIT_NSRC_DECL vmxHCExitIntWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13586	{
13587	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13588
13589	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
13590	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
13591	return vmxHCExitIntWindow(pVCpu, pVmxTransient);
13592	}
13593
13594
13595	/**
13596	* Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
13597	*/
13598	HMVMX_EXIT_NSRC_DECL vmxHCExitNmiWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13599	{
13600	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13601
13602	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
13603	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
13604	return vmxHCExitIntWindow(pVCpu, pVmxTransient);
13605	}
13606
13607
13608	/**
13609	* Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
13610	* Unconditional VM-exit.
13611	*/
13612	HMVMX_EXIT_DECL vmxHCExitTaskSwitchNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13613	{
13614	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13615
13616	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13617	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13618	vmxHCReadIdtVectoringInfoVmcs(pVCpu, pVmxTransient);
13619	vmxHCReadIdtVectoringErrorCodeVmcs(pVCpu, pVmxTransient);
13620
13621	VMXVEXITINFO ExitInfo;
13622	RT_ZERO(ExitInfo);
13623	ExitInfo.uReason = pVmxTransient->uExitReason;
13624	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13625	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13626
13627	VMXVEXITEVENTINFO ExitEventInfo;
13628	RT_ZERO(ExitEventInfo);
13629	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
13630	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
13631	return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
13632	}
13633
13634
13635	/**
13636	* Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
13637	*/
13638	HMVMX_EXIT_DECL vmxHCExitHltNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13639	{
13640	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13641
13642	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
13643	{
13644	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13645	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
13646	}
13647	return vmxHCExitHlt(pVCpu, pVmxTransient);
13648	}
13649
13650
13651	/**
13652	* Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
13653	*/
13654	HMVMX_EXIT_DECL vmxHCExitInvlpgNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13655	{
13656	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13657
13658	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
13659	{
13660	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13661	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13662
13663	VMXVEXITINFO ExitInfo;
13664	RT_ZERO(ExitInfo);
13665	ExitInfo.uReason = pVmxTransient->uExitReason;
13666	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13667	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13668	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
13669	}
13670	return vmxHCExitInvlpg(pVCpu, pVmxTransient);
13671	}
13672
13673
13674	/**
13675	* Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
13676	*/
13677	HMVMX_EXIT_DECL vmxHCExitRdpmcNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13678	{
13679	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13680
13681	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
13682	{
13683	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13684	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
13685	}
13686	return vmxHCExitRdpmc(pVCpu, pVmxTransient);
13687	}
13688
13689
13690	/**
13691	* Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
13692	* (VMX_EXIT_VMWRITE). Conditional VM-exit.
13693	*/
13694	HMVMX_EXIT_DECL vmxHCExitVmreadVmwriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13695	{
13696	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13697
13698	Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD
13699	\|\| pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
13700
13701	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
13702
13703	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
13704	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
13705	uint64_t u64VmcsField = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
13706
13707	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
13708	if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
13709	u64VmcsField &= UINT64_C(0xffffffff);
13710
13711	if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64VmcsField))
13712	{
13713	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13714	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13715
13716	VMXVEXITINFO ExitInfo;
13717	RT_ZERO(ExitInfo);
13718	ExitInfo.uReason = pVmxTransient->uExitReason;
13719	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13720	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13721	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
13722	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
13723	}
13724
13725	if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
13726	return vmxHCExitVmread(pVCpu, pVmxTransient);
13727	return vmxHCExitVmwrite(pVCpu, pVmxTransient);
13728	}
13729
13730
13731	/**
13732	* Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
13733	*/
13734	HMVMX_EXIT_DECL vmxHCExitRdtscNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13735	{
13736	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13737
13738	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
13739	{
13740	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13741	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
13742	}
13743
13744	return vmxHCExitRdtsc(pVCpu, pVmxTransient);
13745	}
13746
13747
13748	/**
13749	* Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
13750	* Conditional VM-exit.
13751	*/
13752	HMVMX_EXIT_DECL vmxHCExitMovCRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13753	{
13754	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13755
13756	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13757	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13758
13759	VBOXSTRICTRC rcStrict;
13760	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
13761	switch (uAccessType)
13762	{
13763	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
13764	{
13765	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
13766	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
13767	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
13768	uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
13769
13770	bool fIntercept;
13771	switch (iCrReg)
13772	{
13773	case 0:
13774	case 4:
13775	fIntercept = CPUMIsGuestVmxMovToCr0Cr4InterceptSet(&pVCpu->cpum.GstCtx, iCrReg, uNewCrX);
13776	break;
13777
13778	case 3:
13779	fIntercept = CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCrX);
13780	break;
13781
13782	case 8:
13783	fIntercept = CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_CR8_LOAD_EXIT);
13784	break;
13785
13786	default:
13787	fIntercept = false;
13788	break;
13789	}
13790	if (fIntercept)
13791	{
13792	VMXVEXITINFO ExitInfo;
13793	RT_ZERO(ExitInfo);
13794	ExitInfo.uReason = pVmxTransient->uExitReason;
13795	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13796	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13797	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
13798	}
13799	else
13800	{
13801	int const rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13802	AssertRCReturn(rc, rc);
13803	rcStrict = vmxHCExitMovToCrX(pVCpu, pVmxTransient->cbExitInstr, iGReg, iCrReg);
13804	}
13805	break;
13806	}
13807
13808	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
13809	{
13810	/*
13811	* CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
13812	* CR2 reads do not cause a VM-exit.
13813	* CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
13814	* CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
13815	*/
13816	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
13817	if ( iCrReg == 3
13818	\|\| iCrReg == 8)
13819	{
13820	static const uint32_t s_auCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
13821	0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
13822	uint32_t const uIntercept = s_auCrXReadIntercepts[iCrReg];
13823	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, uIntercept))
13824	{
13825	VMXVEXITINFO ExitInfo;
13826	RT_ZERO(ExitInfo);
13827	ExitInfo.uReason = pVmxTransient->uExitReason;
13828	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13829	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13830	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
13831	}
13832	else
13833	{
13834	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
13835	rcStrict = vmxHCExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
13836	}
13837	}
13838	else
13839	{
13840	AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
13841	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
13842	}
13843	break;
13844	}
13845
13846	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
13847	{
13848	PCVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
13849	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
13850	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
13851	if ( (uGstHostMask & X86_CR0_TS)
13852	&& (uReadShadow & X86_CR0_TS))
13853	{
13854	VMXVEXITINFO ExitInfo;
13855	RT_ZERO(ExitInfo);
13856	ExitInfo.uReason = pVmxTransient->uExitReason;
13857	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13858	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13859	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
13860	}
13861	else
13862	rcStrict = vmxHCExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr);
13863	break;
13864	}
13865
13866	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
13867	{
13868	RTGCPTR GCPtrEffDst;
13869	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
13870	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
13871	if (fMemOperand)
13872	{
13873	vmxHCReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
13874	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
13875	}
13876	else
13877	GCPtrEffDst = NIL_RTGCPTR;
13878
13879	if (CPUMIsGuestVmxLmswInterceptSet(&pVCpu->cpum.GstCtx, uNewMsw))
13880	{
13881	VMXVEXITINFO ExitInfo;
13882	RT_ZERO(ExitInfo);
13883	ExitInfo.uReason = pVmxTransient->uExitReason;
13884	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13885	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
13886	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13887	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
13888	}
13889	else
13890	rcStrict = vmxHCExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, uNewMsw, GCPtrEffDst);
13891	break;
13892	}
13893
13894	default:
13895	{
13896	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
13897	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
13898	}
13899	}
13900
13901	if (rcStrict == VINF_IEM_RAISED_XCPT)
13902	{
13903	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13904	rcStrict = VINF_SUCCESS;
13905	}
13906	return rcStrict;
13907	}
13908
13909
13910	/**
13911	* Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
13912	* Conditional VM-exit.
13913	*/
13914	HMVMX_EXIT_DECL vmxHCExitMovDRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13915	{
13916	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13917
13918	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
13919	{
13920	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13921	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13922
13923	VMXVEXITINFO ExitInfo;
13924	RT_ZERO(ExitInfo);
13925	ExitInfo.uReason = pVmxTransient->uExitReason;
13926	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13927	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13928	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
13929	}
13930	return vmxHCExitMovDRx(pVCpu, pVmxTransient);
13931	}
13932
13933
13934	/**
13935	* Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
13936	* Conditional VM-exit.
13937	*/
13938	HMVMX_EXIT_DECL vmxHCExitIoInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13939	{
13940	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13941
13942	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13943
13944	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
13945	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
13946	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
13947
13948	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
13949	uint8_t const cbAccess = s_aIOSizes[uIOSize];
13950	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
13951	{
13952	/*
13953	* IN/OUT instruction:
13954	* - Provides VM-exit instruction length.
13955	*
13956	* INS/OUTS instruction:
13957	* - Provides VM-exit instruction length.
13958	* - Provides Guest-linear address.
13959	* - Optionally provides VM-exit instruction info (depends on CPU feature).
13960	*/
13961	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
13962	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13963
13964	/* Make sure we don't use stale/uninitialized VMX-transient info. below. */
13965	pVmxTransient->ExitInstrInfo.u = 0;
13966	pVmxTransient->uGuestLinearAddr = 0;
13967
13968	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
13969	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
13970	if (fIOString)
13971	{
13972	vmxHCReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
13973	if (fVmxInsOutsInfo)
13974	{
13975	Assert(RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
13976	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
13977	}
13978	}
13979
13980	VMXVEXITINFO ExitInfo;
13981	RT_ZERO(ExitInfo);
13982	ExitInfo.uReason = pVmxTransient->uExitReason;
13983	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13984	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13985	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
13986	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
13987	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
13988	}
13989	return vmxHCExitIoInstr(pVCpu, pVmxTransient);
13990	}
13991
13992
13993	/**
13994	* Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
13995	*/
13996	HMVMX_EXIT_DECL vmxHCExitRdmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13997	{
13998	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13999
14000	uint32_t fMsrpm;
14001	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
14002	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap, pVCpu->cpum.GstCtx.ecx);
14003	else
14004	fMsrpm = VMXMSRPM_EXIT_RD;
14005
14006	if (fMsrpm & VMXMSRPM_EXIT_RD)
14007	{
14008	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
14009	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
14010	}
14011	return vmxHCExitRdmsr(pVCpu, pVmxTransient);
14012	}
14013
14014
14015	/**
14016	* Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
14017	*/
14018	HMVMX_EXIT_DECL vmxHCExitWrmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14019	{
14020	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14021
14022	uint32_t fMsrpm;
14023	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
14024	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap, pVCpu->cpum.GstCtx.ecx);
14025	else
14026	fMsrpm = VMXMSRPM_EXIT_WR;
14027
14028	if (fMsrpm & VMXMSRPM_EXIT_WR)
14029	{
14030	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
14031	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
14032	}
14033	return vmxHCExitWrmsr(pVCpu, pVmxTransient);
14034	}
14035
14036
14037	/**
14038	* Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
14039	*/
14040	HMVMX_EXIT_DECL vmxHCExitMwaitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14041	{
14042	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14043
14044	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
14045	{
14046	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
14047	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
14048	}
14049	return vmxHCExitMwait(pVCpu, pVmxTransient);
14050	}
14051
14052
14053	/**
14054	* Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
14055	* VM-exit.
14056	*/
14057	HMVMX_EXIT_DECL vmxHCExitMtfNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14058	{
14059	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14060
14061	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
14062	vmxHCReadGuestPendingDbgXctps(pVCpu, pVmxTransient);
14063	VMXVEXITINFO ExitInfo;
14064	RT_ZERO(ExitInfo);
14065	ExitInfo.uReason = pVmxTransient->uExitReason;
14066	ExitInfo.u64GuestPendingDbgXcpts = pVmxTransient->uGuestPendingDbgXcpts;
14067	return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
14068	}
14069
14070
14071	/**
14072	* Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
14073	*/
14074	HMVMX_EXIT_DECL vmxHCExitMonitorNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14075	{
14076	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14077
14078	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
14079	{
14080	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
14081	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
14082	}
14083	return vmxHCExitMonitor(pVCpu, pVmxTransient);
14084	}
14085
14086
14087	/**
14088	* Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
14089	*/
14090	HMVMX_EXIT_DECL vmxHCExitPauseNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14091	{
14092	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14093
14094	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
14095	* PAUSE when executing a nested-guest? If it does not, we would not need
14096	* to check for the intercepts here. Just call VM-exit... */
14097
14098	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
14099	if ( CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
14100	\|\| CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
14101	{
14102	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
14103	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
14104	}
14105	return vmxHCExitPause(pVCpu, pVmxTransient);
14106	}
14107
14108
14109	/**
14110	* Nested-guest VM-exit handler for when the TPR value is lowered below the
14111	* specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
14112	*/
14113	HMVMX_EXIT_NSRC_DECL vmxHCExitTprBelowThresholdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14114	{
14115	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14116
14117	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
14118	{
14119	vmxHCReadGuestPendingDbgXctps(pVCpu, pVmxTransient);
14120	VMXVEXITINFO ExitInfo;
14121	RT_ZERO(ExitInfo);
14122	ExitInfo.uReason = pVmxTransient->uExitReason;
14123	ExitInfo.u64GuestPendingDbgXcpts = pVmxTransient->uGuestPendingDbgXcpts;
14124	return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
14125	}
14126	return vmxHCExitTprBelowThreshold(pVCpu, pVmxTransient);
14127	}
14128
14129
14130	/**
14131	* Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
14132	* VM-exit.
14133	*/
14134	HMVMX_EXIT_DECL vmxHCExitApicAccessNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14135	{
14136	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14137
14138	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
14139	vmxHCReadIdtVectoringInfoVmcs(pVCpu, pVmxTransient);
14140	vmxHCReadIdtVectoringErrorCodeVmcs(pVCpu, pVmxTransient);
14141	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
14142
14143	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
14144
14145	Log4Func(("at offset %#x type=%u\n", VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual),
14146	VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual)));
14147
14148	VMXVEXITINFO ExitInfo;
14149	RT_ZERO(ExitInfo);
14150	ExitInfo.uReason = pVmxTransient->uExitReason;
14151	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
14152	ExitInfo.u64Qual = pVmxTransient->uExitQual;
14153
14154	VMXVEXITEVENTINFO ExitEventInfo;
14155	RT_ZERO(ExitEventInfo);
14156	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
14157	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
14158	return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
14159	}
14160
14161
14162	/**
14163	* Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
14164	* Conditional VM-exit.
14165	*/
14166	HMVMX_EXIT_DECL vmxHCExitApicWriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14167	{
14168	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14169
14170	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
14171	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
14172	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
14173	}
14174
14175
14176	/**
14177	* Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
14178	* Conditional VM-exit.
14179	*/
14180	HMVMX_EXIT_DECL vmxHCExitVirtEoiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14181	{
14182	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14183
14184	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
14185	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
14186	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
14187	}
14188
14189
14190	/**
14191	* Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
14192	*/
14193	HMVMX_EXIT_DECL vmxHCExitRdtscpNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14194	{
14195	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14196
14197	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
14198	{
14199	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
14200	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
14201	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
14202	}
14203	return vmxHCExitRdtscp(pVCpu, pVmxTransient);
14204	}
14205
14206
14207	/**
14208	* Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
14209	*/
14210	HMVMX_EXIT_NSRC_DECL vmxHCExitWbinvdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14211	{
14212	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14213
14214	if (CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
14215	{
14216	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
14217	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
14218	}
14219	return vmxHCExitWbinvd(pVCpu, pVmxTransient);
14220	}
14221
14222
14223	/**
14224	* Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
14225	*/
14226	HMVMX_EXIT_DECL vmxHCExitInvpcidNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14227	{
14228	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14229
14230	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
14231	{
14232	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
14233	vmxHCReadExitInstrLenVmcs(ppVCpu, VmxTransient);
14234	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
14235	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
14236
14237	VMXVEXITINFO ExitInfo;
14238	RT_ZERO(ExitInfo);
14239	ExitInfo.uReason = pVmxTransient->uExitReason;
14240	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
14241	ExitInfo.u64Qual = pVmxTransient->uExitQual;
14242	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
14243	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
14244	}
14245	return vmxHCExitInvpcid(pVCpu, pVmxTransient);
14246	}
14247
14248
14249	/**
14250	* Nested-guest VM-exit handler for invalid-guest state
14251	* (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
14252	*/
14253	HMVMX_EXIT_DECL vmxHCExitErrInvalidGuestStateNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14254	{
14255	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14256
14257	/*
14258	* Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
14259	* So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
14260	* Handle it like it's in an invalid guest state of the outer guest.
14261	*
14262	* When the fast path is implemented, this should be changed to cause the corresponding
14263	* nested-guest VM-exit.
14264	*/
14265	return vmxHCExitErrInvalidGuestState(pVCpu, pVmxTransient);
14266	}
14267
14268
14269	/**
14270	* Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
14271	* and only provide the instruction length.
14272	*
14273	* Unconditional VM-exit.
14274	*/
14275	HMVMX_EXIT_DECL vmxHCExitInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14276	{
14277	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14278
14279	#ifdef VBOX_STRICT
14280	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14281	switch (pVmxTransient->uExitReason)
14282	{
14283	case VMX_EXIT_ENCLS:
14284	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
14285	break;
14286
14287	case VMX_EXIT_VMFUNC:
14288	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_VMFUNC));
14289	break;
14290	}
14291	#endif
14292
14293	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
14294	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
14295	}
14296
14297
14298	/**
14299	* Nested-guest VM-exit handler for instructions that provide instruction length as
14300	* well as more information.
14301	*
14302	* Unconditional VM-exit.
14303	*/
14304	HMVMX_EXIT_DECL vmxHCExitInstrWithInfoNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14305	{
14306	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14307
14308	#ifdef VBOX_STRICT
14309	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14310	switch (pVmxTransient->uExitReason)
14311	{
14312	case VMX_EXIT_GDTR_IDTR_ACCESS:
14313	case VMX_EXIT_LDTR_TR_ACCESS:
14314	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
14315	break;
14316
14317	case VMX_EXIT_RDRAND:
14318	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
14319	break;
14320
14321	case VMX_EXIT_RDSEED:
14322	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
14323	break;
14324
14325	case VMX_EXIT_XSAVES:
14326	case VMX_EXIT_XRSTORS:
14327	/** @todo NSTVMX: Verify XSS-bitmap. */
14328	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
14329	break;
14330
14331	case VMX_EXIT_UMWAIT:
14332	case VMX_EXIT_TPAUSE:
14333	Assert(CPUMIsGuestVmxProcCtlsSet(pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
14334	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
14335	break;
14336
14337	case VMX_EXIT_LOADIWKEY:
14338	Assert(CPUMIsGuestVmxProcCtls3Set(pCtx, VMX_PROC_CTLS3_LOADIWKEY_EXIT));
14339	break;
14340	}
14341	#endif
14342
14343	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
14344	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
14345	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
14346
14347	VMXVEXITINFO ExitInfo;
14348	RT_ZERO(ExitInfo);
14349	ExitInfo.uReason = pVmxTransient->uExitReason;
14350	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
14351	ExitInfo.u64Qual = pVmxTransient->uExitQual;
14352	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
14353	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
14354	}
14355
14356	/** @} */
14357	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
14358

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/VMXAllTemplate.cpp.h@ 92451

Download in other formats: