VMXAllTemplate.cpp.h@ 97060

Last change on this file since 97060 was 97060, checked in by vboxsync, 3 years ago
VMM/HMVMXR0: Changed vmxHCImportGuestIntrState into a template, since the a_iSegReg argument is constant in all calls and marked other vmxHCImportGuestState helpers as DECLINLINE just to formalize what the compilers probably are doing already.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 496.4 KB

Line
1	/* $Id: VMXAllTemplate.cpp.h 97060 2022-10-09 22:20:19Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Code template for our own hypervisor and the NEM darwin backend using Apple's Hypervisor.framework.
4	*/
5
6	/*
7	* Copyright (C) 2012-2022 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Defined Constants And Macros *
31	*********************************************************************************************************************************/
32	#if !defined(VMX_VMCS_WRITE_16) \|\| !defined(VMX_VMCS_WRITE_32) \|\| !defined(VMX_VMCS_WRITE_64) \|\| !defined(VMX_VMCS_WRITE_64)
33	# error "At least one of the VMX_VMCS_WRITE_16, VMX_VMCS_WRITE_32, VMX_VMCS_WRITE_64 or VMX_VMCS_WRITE_64 is missing"
34	#endif
35
36
37	#if !defined(VMX_VMCS_READ_16) \|\| !defined(VMX_VMCS_READ_32) \|\| !defined(VMX_VMCS_READ_64) \|\| !defined(VMX_VMCS_READ_64)
38	# error "At least one of the VMX_VMCS_READ_16, VMX_VMCS_READ_32, VMX_VMCS_READ_64 or VMX_VMCS_READ_64 is missing"
39	#endif
40
41	/** Enables condensing of VMREAD instructions, see vmxHCReadToTransient(). */
42	#define HMVMX_WITH_CONDENSED_VMREADS
43
44	/** Use the function table. */
45	#define HMVMX_USE_FUNCTION_TABLE
46
47	/** Determine which tagged-TLB flush handler to use. */
48	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
49	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
50	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
51	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
52
53	/** Assert that all the given fields have been read from the VMCS. */
54	#ifdef VBOX_STRICT
55	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) \
56	do { \
57	uint32_t const fVmcsFieldRead = ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead); \
58	Assert((fVmcsFieldRead & (a_fReadFields)) == (a_fReadFields)); \
59	} while (0)
60	#else
61	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) do { } while (0)
62	#endif
63
64	/**
65	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
66	* guest using hardware-assisted VMX.
67	*
68	* This excludes state like GPRs (other than RSP) which are always are
69	* swapped and restored across the world-switch and also registers like EFER,
70	* MSR which cannot be modified by the guest without causing a VM-exit.
71	*/
72	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
73	\| CPUMCTX_EXTRN_RFLAGS \
74	\| CPUMCTX_EXTRN_RSP \
75	\| CPUMCTX_EXTRN_SREG_MASK \
76	\| CPUMCTX_EXTRN_TABLE_MASK \
77	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
78	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
79	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
80	\| CPUMCTX_EXTRN_TSC_AUX \
81	\| CPUMCTX_EXTRN_OTHER_MSRS \
82	\| CPUMCTX_EXTRN_CR0 \
83	\| CPUMCTX_EXTRN_CR3 \
84	\| CPUMCTX_EXTRN_CR4 \
85	\| CPUMCTX_EXTRN_DR7 \
86	\| CPUMCTX_EXTRN_HWVIRT \
87	\| CPUMCTX_EXTRN_INHIBIT_INT \
88	\| CPUMCTX_EXTRN_INHIBIT_NMI)
89
90	/**
91	* Exception bitmap mask for real-mode guests (real-on-v86).
92	*
93	* We need to intercept all exceptions manually except:
94	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
95	* due to bugs in Intel CPUs.
96	* - \#PF need not be intercepted even in real-mode if we have nested paging
97	* support.
98	*/
99	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
100	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
101	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
102	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
103	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
104	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
105	\| RT_BIT(X86_XCPT_XF))
106
107	/** Maximum VM-instruction error number. */
108	#define HMVMX_INSTR_ERROR_MAX 28
109
110	/** Profiling macro. */
111	#ifdef HM_PROFILE_EXIT_DISPATCH
112	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitDispatch, ed)
113	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitDispatch, ed)
114	#else
115	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
116	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
117	#endif
118
119	#ifndef IN_NEM_DARWIN
120	/** Assert that preemption is disabled or covered by thread-context hooks. */
121	# define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
122	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
123
124	/** Assert that we haven't migrated CPUs when thread-context hooks are not
125	* used. */
126	# define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
127	\|\| (a_pVCpu)->hmr0.s.idEnteredCpu == RTMpCpuId(), \
128	("Illegal migration! Entered on CPU %u Current %u\n", \
129	(a_pVCpu)->hmr0.s.idEnteredCpu, RTMpCpuId()))
130	#else
131	# define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) do { } while (0)
132	# define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) do { } while (0)
133	#endif
134
135	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
136	* context. */
137	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
138	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
139	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
140
141	/** Log the VM-exit reason with an easily visible marker to identify it in a
142	* potential sea of logging data. */
143	#define HMVMX_LOG_EXIT(a_pVCpu, a_uExitReason) \
144	do { \
145	Log4(("VM-exit: vcpu[%RU32] %85s -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu, \
146	HMGetVmxExitName(a_uExitReason))); \
147	} while (0) \
148
149
150	/*********************************************************************************************************************************
151	* Structures and Typedefs *
152	*********************************************************************************************************************************/
153	/**
154	* Memory operand read or write access.
155	*/
156	typedef enum VMXMEMACCESS
157	{
158	VMXMEMACCESS_READ = 0,
159	VMXMEMACCESS_WRITE = 1
160	} VMXMEMACCESS;
161
162
163	/**
164	* VMX VM-exit handler.
165	*
166	* @returns Strict VBox status code (i.e. informational status codes too).
167	* @param pVCpu The cross context virtual CPU structure.
168	* @param pVmxTransient The VMX-transient structure.
169	*/
170	#ifndef HMVMX_USE_FUNCTION_TABLE
171	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
172	#else
173	typedef DECLCALLBACKTYPE(VBOXSTRICTRC, FNVMXEXITHANDLER,(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient));
174	/** Pointer to VM-exit handler. */
175	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
176	#endif
177
178	/**
179	* VMX VM-exit handler, non-strict status code.
180	*
181	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
182	*
183	* @returns VBox status code, no informational status code returned.
184	* @param pVCpu The cross context virtual CPU structure.
185	* @param pVmxTransient The VMX-transient structure.
186	*
187	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
188	* use of that status code will be replaced with VINF_EM_SOMETHING
189	* later when switching over to IEM.
190	*/
191	#ifndef HMVMX_USE_FUNCTION_TABLE
192	typedef int FNVMXEXITHANDLERNSRC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
193	#else
194	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
195	#endif
196
197
198	/*********************************************************************************************************************************
199	* Internal Functions *
200	*********************************************************************************************************************************/
201	#ifndef HMVMX_USE_FUNCTION_TABLE
202	DECLINLINE(VBOXSTRICTRC) vmxHCHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
203	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
204	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
205	#else
206	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
207	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
208	#endif
209	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
210	DECLINLINE(VBOXSTRICTRC) vmxHCHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
211	#endif
212
213	static int vmxHCImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
214
215	/** @name VM-exit handler prototypes.
216	* @{
217	*/
218	static FNVMXEXITHANDLER vmxHCExitXcptOrNmi;
219	static FNVMXEXITHANDLER vmxHCExitExtInt;
220	static FNVMXEXITHANDLER vmxHCExitTripleFault;
221	static FNVMXEXITHANDLERNSRC vmxHCExitIntWindow;
222	static FNVMXEXITHANDLERNSRC vmxHCExitNmiWindow;
223	static FNVMXEXITHANDLER vmxHCExitTaskSwitch;
224	static FNVMXEXITHANDLER vmxHCExitCpuid;
225	static FNVMXEXITHANDLER vmxHCExitGetsec;
226	static FNVMXEXITHANDLER vmxHCExitHlt;
227	static FNVMXEXITHANDLERNSRC vmxHCExitInvd;
228	static FNVMXEXITHANDLER vmxHCExitInvlpg;
229	static FNVMXEXITHANDLER vmxHCExitRdpmc;
230	static FNVMXEXITHANDLER vmxHCExitVmcall;
231	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
232	static FNVMXEXITHANDLER vmxHCExitVmclear;
233	static FNVMXEXITHANDLER vmxHCExitVmlaunch;
234	static FNVMXEXITHANDLER vmxHCExitVmptrld;
235	static FNVMXEXITHANDLER vmxHCExitVmptrst;
236	static FNVMXEXITHANDLER vmxHCExitVmread;
237	static FNVMXEXITHANDLER vmxHCExitVmresume;
238	static FNVMXEXITHANDLER vmxHCExitVmwrite;
239	static FNVMXEXITHANDLER vmxHCExitVmxoff;
240	static FNVMXEXITHANDLER vmxHCExitVmxon;
241	static FNVMXEXITHANDLER vmxHCExitInvvpid;
242	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
243	static FNVMXEXITHANDLER vmxHCExitInvept;
244	# endif
245	#endif
246	static FNVMXEXITHANDLER vmxHCExitRdtsc;
247	static FNVMXEXITHANDLER vmxHCExitMovCRx;
248	static FNVMXEXITHANDLER vmxHCExitMovDRx;
249	static FNVMXEXITHANDLER vmxHCExitIoInstr;
250	static FNVMXEXITHANDLER vmxHCExitRdmsr;
251	static FNVMXEXITHANDLER vmxHCExitWrmsr;
252	static FNVMXEXITHANDLER vmxHCExitMwait;
253	static FNVMXEXITHANDLER vmxHCExitMtf;
254	static FNVMXEXITHANDLER vmxHCExitMonitor;
255	static FNVMXEXITHANDLER vmxHCExitPause;
256	static FNVMXEXITHANDLERNSRC vmxHCExitTprBelowThreshold;
257	static FNVMXEXITHANDLER vmxHCExitApicAccess;
258	static FNVMXEXITHANDLER vmxHCExitEptViolation;
259	static FNVMXEXITHANDLER vmxHCExitEptMisconfig;
260	static FNVMXEXITHANDLER vmxHCExitRdtscp;
261	static FNVMXEXITHANDLER vmxHCExitPreemptTimer;
262	static FNVMXEXITHANDLERNSRC vmxHCExitWbinvd;
263	static FNVMXEXITHANDLER vmxHCExitXsetbv;
264	static FNVMXEXITHANDLER vmxHCExitInvpcid;
265	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
266	static FNVMXEXITHANDLERNSRC vmxHCExitSetPendingXcptUD;
267	#endif
268	static FNVMXEXITHANDLERNSRC vmxHCExitErrInvalidGuestState;
269	static FNVMXEXITHANDLERNSRC vmxHCExitErrUnexpected;
270	/** @} */
271
272	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
273	/** @name Nested-guest VM-exit handler prototypes.
274	* @{
275	*/
276	static FNVMXEXITHANDLER vmxHCExitXcptOrNmiNested;
277	static FNVMXEXITHANDLER vmxHCExitTripleFaultNested;
278	static FNVMXEXITHANDLERNSRC vmxHCExitIntWindowNested;
279	static FNVMXEXITHANDLERNSRC vmxHCExitNmiWindowNested;
280	static FNVMXEXITHANDLER vmxHCExitTaskSwitchNested;
281	static FNVMXEXITHANDLER vmxHCExitHltNested;
282	static FNVMXEXITHANDLER vmxHCExitInvlpgNested;
283	static FNVMXEXITHANDLER vmxHCExitRdpmcNested;
284	static FNVMXEXITHANDLER vmxHCExitVmreadVmwriteNested;
285	static FNVMXEXITHANDLER vmxHCExitRdtscNested;
286	static FNVMXEXITHANDLER vmxHCExitMovCRxNested;
287	static FNVMXEXITHANDLER vmxHCExitMovDRxNested;
288	static FNVMXEXITHANDLER vmxHCExitIoInstrNested;
289	static FNVMXEXITHANDLER vmxHCExitRdmsrNested;
290	static FNVMXEXITHANDLER vmxHCExitWrmsrNested;
291	static FNVMXEXITHANDLER vmxHCExitMwaitNested;
292	static FNVMXEXITHANDLER vmxHCExitMtfNested;
293	static FNVMXEXITHANDLER vmxHCExitMonitorNested;
294	static FNVMXEXITHANDLER vmxHCExitPauseNested;
295	static FNVMXEXITHANDLERNSRC vmxHCExitTprBelowThresholdNested;
296	static FNVMXEXITHANDLER vmxHCExitApicAccessNested;
297	static FNVMXEXITHANDLER vmxHCExitApicWriteNested;
298	static FNVMXEXITHANDLER vmxHCExitVirtEoiNested;
299	static FNVMXEXITHANDLER vmxHCExitRdtscpNested;
300	static FNVMXEXITHANDLERNSRC vmxHCExitWbinvdNested;
301	static FNVMXEXITHANDLER vmxHCExitInvpcidNested;
302	static FNVMXEXITHANDLERNSRC vmxHCExitErrInvalidGuestStateNested;
303	static FNVMXEXITHANDLER vmxHCExitInstrNested;
304	static FNVMXEXITHANDLER vmxHCExitInstrWithInfoNested;
305	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
306	static FNVMXEXITHANDLER vmxHCExitEptViolationNested;
307	static FNVMXEXITHANDLER vmxHCExitEptMisconfigNested;
308	# endif
309	/** @} */
310	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
311
312
313	/*********************************************************************************************************************************
314	* Global Variables *
315	*********************************************************************************************************************************/
316	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
317	/**
318	* Array of all VMCS fields.
319	* Any fields added to the VT-x spec. should be added here.
320	*
321	* Currently only used to derive shadow VMCS fields for hardware-assisted execution
322	* of nested-guests.
323	*/
324	static const uint32_t g_aVmcsFields[] =
325	{
326	/* 16-bit control fields. */
327	VMX_VMCS16_VPID,
328	VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR,
329	VMX_VMCS16_EPTP_INDEX,
330
331	/* 16-bit guest-state fields. */
332	VMX_VMCS16_GUEST_ES_SEL,
333	VMX_VMCS16_GUEST_CS_SEL,
334	VMX_VMCS16_GUEST_SS_SEL,
335	VMX_VMCS16_GUEST_DS_SEL,
336	VMX_VMCS16_GUEST_FS_SEL,
337	VMX_VMCS16_GUEST_GS_SEL,
338	VMX_VMCS16_GUEST_LDTR_SEL,
339	VMX_VMCS16_GUEST_TR_SEL,
340	VMX_VMCS16_GUEST_INTR_STATUS,
341	VMX_VMCS16_GUEST_PML_INDEX,
342
343	/* 16-bits host-state fields. */
344	VMX_VMCS16_HOST_ES_SEL,
345	VMX_VMCS16_HOST_CS_SEL,
346	VMX_VMCS16_HOST_SS_SEL,
347	VMX_VMCS16_HOST_DS_SEL,
348	VMX_VMCS16_HOST_FS_SEL,
349	VMX_VMCS16_HOST_GS_SEL,
350	VMX_VMCS16_HOST_TR_SEL,
351
352	/* 64-bit control fields. */
353	VMX_VMCS64_CTRL_IO_BITMAP_A_FULL,
354	VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH,
355	VMX_VMCS64_CTRL_IO_BITMAP_B_FULL,
356	VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH,
357	VMX_VMCS64_CTRL_MSR_BITMAP_FULL,
358	VMX_VMCS64_CTRL_MSR_BITMAP_HIGH,
359	VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL,
360	VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH,
361	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,
362	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH,
363	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL,
364	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH,
365	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL,
366	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH,
367	VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL,
368	VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH,
369	VMX_VMCS64_CTRL_TSC_OFFSET_FULL,
370	VMX_VMCS64_CTRL_TSC_OFFSET_HIGH,
371	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL,
372	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH,
373	VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL,
374	VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH,
375	VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL,
376	VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH,
377	VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL,
378	VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH,
379	VMX_VMCS64_CTRL_EPTP_FULL,
380	VMX_VMCS64_CTRL_EPTP_HIGH,
381	VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL,
382	VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH,
383	VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL,
384	VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH,
385	VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL,
386	VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH,
387	VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL,
388	VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH,
389	VMX_VMCS64_CTRL_EPTP_LIST_FULL,
390	VMX_VMCS64_CTRL_EPTP_LIST_HIGH,
391	VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL,
392	VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH,
393	VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL,
394	VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH,
395	VMX_VMCS64_CTRL_VE_XCPT_INFO_ADDR_FULL,
396	VMX_VMCS64_CTRL_VE_XCPT_INFO_ADDR_HIGH,
397	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL,
398	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH,
399	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL,
400	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH,
401	VMX_VMCS64_CTRL_SPPTP_FULL,
402	VMX_VMCS64_CTRL_SPPTP_HIGH,
403	VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL,
404	VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH,
405	VMX_VMCS64_CTRL_PROC_EXEC3_FULL,
406	VMX_VMCS64_CTRL_PROC_EXEC3_HIGH,
407	VMX_VMCS64_CTRL_ENCLV_EXITING_BITMAP_FULL,
408	VMX_VMCS64_CTRL_ENCLV_EXITING_BITMAP_HIGH,
409
410	/* 64-bit read-only data fields. */
411	VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,
412	VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH,
413
414	/* 64-bit guest-state fields. */
415	VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL,
416	VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH,
417	VMX_VMCS64_GUEST_DEBUGCTL_FULL,
418	VMX_VMCS64_GUEST_DEBUGCTL_HIGH,
419	VMX_VMCS64_GUEST_PAT_FULL,
420	VMX_VMCS64_GUEST_PAT_HIGH,
421	VMX_VMCS64_GUEST_EFER_FULL,
422	VMX_VMCS64_GUEST_EFER_HIGH,
423	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL,
424	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH,
425	VMX_VMCS64_GUEST_PDPTE0_FULL,
426	VMX_VMCS64_GUEST_PDPTE0_HIGH,
427	VMX_VMCS64_GUEST_PDPTE1_FULL,
428	VMX_VMCS64_GUEST_PDPTE1_HIGH,
429	VMX_VMCS64_GUEST_PDPTE2_FULL,
430	VMX_VMCS64_GUEST_PDPTE2_HIGH,
431	VMX_VMCS64_GUEST_PDPTE3_FULL,
432	VMX_VMCS64_GUEST_PDPTE3_HIGH,
433	VMX_VMCS64_GUEST_BNDCFGS_FULL,
434	VMX_VMCS64_GUEST_BNDCFGS_HIGH,
435	VMX_VMCS64_GUEST_RTIT_CTL_FULL,
436	VMX_VMCS64_GUEST_RTIT_CTL_HIGH,
437	VMX_VMCS64_GUEST_PKRS_FULL,
438	VMX_VMCS64_GUEST_PKRS_HIGH,
439
440	/* 64-bit host-state fields. */
441	VMX_VMCS64_HOST_PAT_FULL,
442	VMX_VMCS64_HOST_PAT_HIGH,
443	VMX_VMCS64_HOST_EFER_FULL,
444	VMX_VMCS64_HOST_EFER_HIGH,
445	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL,
446	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH,
447	VMX_VMCS64_HOST_PKRS_FULL,
448	VMX_VMCS64_HOST_PKRS_HIGH,
449
450	/* 32-bit control fields. */
451	VMX_VMCS32_CTRL_PIN_EXEC,
452	VMX_VMCS32_CTRL_PROC_EXEC,
453	VMX_VMCS32_CTRL_EXCEPTION_BITMAP,
454	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,
455	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,
456	VMX_VMCS32_CTRL_CR3_TARGET_COUNT,
457	VMX_VMCS32_CTRL_EXIT,
458	VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,
459	VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,
460	VMX_VMCS32_CTRL_ENTRY,
461	VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,
462	VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,
463	VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,
464	VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,
465	VMX_VMCS32_CTRL_TPR_THRESHOLD,
466	VMX_VMCS32_CTRL_PROC_EXEC2,
467	VMX_VMCS32_CTRL_PLE_GAP,
468	VMX_VMCS32_CTRL_PLE_WINDOW,
469
470	/* 32-bits read-only fields. */
471	VMX_VMCS32_RO_VM_INSTR_ERROR,
472	VMX_VMCS32_RO_EXIT_REASON,
473	VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,
474	VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE,
475	VMX_VMCS32_RO_IDT_VECTORING_INFO,
476	VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,
477	VMX_VMCS32_RO_EXIT_INSTR_LENGTH,
478	VMX_VMCS32_RO_EXIT_INSTR_INFO,
479
480	/* 32-bit guest-state fields. */
481	VMX_VMCS32_GUEST_ES_LIMIT,
482	VMX_VMCS32_GUEST_CS_LIMIT,
483	VMX_VMCS32_GUEST_SS_LIMIT,
484	VMX_VMCS32_GUEST_DS_LIMIT,
485	VMX_VMCS32_GUEST_FS_LIMIT,
486	VMX_VMCS32_GUEST_GS_LIMIT,
487	VMX_VMCS32_GUEST_LDTR_LIMIT,
488	VMX_VMCS32_GUEST_TR_LIMIT,
489	VMX_VMCS32_GUEST_GDTR_LIMIT,
490	VMX_VMCS32_GUEST_IDTR_LIMIT,
491	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
492	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
493	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
494	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
495	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
496	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS,
497	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS,
498	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS,
499	VMX_VMCS32_GUEST_INT_STATE,
500	VMX_VMCS32_GUEST_ACTIVITY_STATE,
501	VMX_VMCS32_GUEST_SMBASE,
502	VMX_VMCS32_GUEST_SYSENTER_CS,
503	VMX_VMCS32_PREEMPT_TIMER_VALUE,
504
505	/* 32-bit host-state fields. */
506	VMX_VMCS32_HOST_SYSENTER_CS,
507
508	/* Natural-width control fields. */
509	VMX_VMCS_CTRL_CR0_MASK,
510	VMX_VMCS_CTRL_CR4_MASK,
511	VMX_VMCS_CTRL_CR0_READ_SHADOW,
512	VMX_VMCS_CTRL_CR4_READ_SHADOW,
513	VMX_VMCS_CTRL_CR3_TARGET_VAL0,
514	VMX_VMCS_CTRL_CR3_TARGET_VAL1,
515	VMX_VMCS_CTRL_CR3_TARGET_VAL2,
516	VMX_VMCS_CTRL_CR3_TARGET_VAL3,
517
518	/* Natural-width read-only data fields. */
519	VMX_VMCS_RO_EXIT_QUALIFICATION,
520	VMX_VMCS_RO_IO_RCX,
521	VMX_VMCS_RO_IO_RSI,
522	VMX_VMCS_RO_IO_RDI,
523	VMX_VMCS_RO_IO_RIP,
524	VMX_VMCS_RO_GUEST_LINEAR_ADDR,
525
526	/* Natural-width guest-state field */
527	VMX_VMCS_GUEST_CR0,
528	VMX_VMCS_GUEST_CR3,
529	VMX_VMCS_GUEST_CR4,
530	VMX_VMCS_GUEST_ES_BASE,
531	VMX_VMCS_GUEST_CS_BASE,
532	VMX_VMCS_GUEST_SS_BASE,
533	VMX_VMCS_GUEST_DS_BASE,
534	VMX_VMCS_GUEST_FS_BASE,
535	VMX_VMCS_GUEST_GS_BASE,
536	VMX_VMCS_GUEST_LDTR_BASE,
537	VMX_VMCS_GUEST_TR_BASE,
538	VMX_VMCS_GUEST_GDTR_BASE,
539	VMX_VMCS_GUEST_IDTR_BASE,
540	VMX_VMCS_GUEST_DR7,
541	VMX_VMCS_GUEST_RSP,
542	VMX_VMCS_GUEST_RIP,
543	VMX_VMCS_GUEST_RFLAGS,
544	VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
545	VMX_VMCS_GUEST_SYSENTER_ESP,
546	VMX_VMCS_GUEST_SYSENTER_EIP,
547	VMX_VMCS_GUEST_S_CET,
548	VMX_VMCS_GUEST_SSP,
549	VMX_VMCS_GUEST_INTR_SSP_TABLE_ADDR,
550
551	/* Natural-width host-state fields */
552	VMX_VMCS_HOST_CR0,
553	VMX_VMCS_HOST_CR3,
554	VMX_VMCS_HOST_CR4,
555	VMX_VMCS_HOST_FS_BASE,
556	VMX_VMCS_HOST_GS_BASE,
557	VMX_VMCS_HOST_TR_BASE,
558	VMX_VMCS_HOST_GDTR_BASE,
559	VMX_VMCS_HOST_IDTR_BASE,
560	VMX_VMCS_HOST_SYSENTER_ESP,
561	VMX_VMCS_HOST_SYSENTER_EIP,
562	VMX_VMCS_HOST_RSP,
563	VMX_VMCS_HOST_RIP,
564	VMX_VMCS_HOST_S_CET,
565	VMX_VMCS_HOST_SSP,
566	VMX_VMCS_HOST_INTR_SSP_TABLE_ADDR
567	};
568	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
569
570	#ifdef VBOX_STRICT
571	static const uint32_t g_aVmcsSegBase[] =
572	{
573	VMX_VMCS_GUEST_ES_BASE,
574	VMX_VMCS_GUEST_CS_BASE,
575	VMX_VMCS_GUEST_SS_BASE,
576	VMX_VMCS_GUEST_DS_BASE,
577	VMX_VMCS_GUEST_FS_BASE,
578	VMX_VMCS_GUEST_GS_BASE
579	};
580	static const uint32_t g_aVmcsSegSel[] =
581	{
582	VMX_VMCS16_GUEST_ES_SEL,
583	VMX_VMCS16_GUEST_CS_SEL,
584	VMX_VMCS16_GUEST_SS_SEL,
585	VMX_VMCS16_GUEST_DS_SEL,
586	VMX_VMCS16_GUEST_FS_SEL,
587	VMX_VMCS16_GUEST_GS_SEL
588	};
589	static const uint32_t g_aVmcsSegLimit[] =
590	{
591	VMX_VMCS32_GUEST_ES_LIMIT,
592	VMX_VMCS32_GUEST_CS_LIMIT,
593	VMX_VMCS32_GUEST_SS_LIMIT,
594	VMX_VMCS32_GUEST_DS_LIMIT,
595	VMX_VMCS32_GUEST_FS_LIMIT,
596	VMX_VMCS32_GUEST_GS_LIMIT
597	};
598	static const uint32_t g_aVmcsSegAttr[] =
599	{
600	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
601	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
602	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
603	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
604	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
605	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
606	};
607	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
608	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
609	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
610	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
611	#endif /* VBOX_STRICT */
612
613	#ifdef HMVMX_USE_FUNCTION_TABLE
614	/**
615	* VMX_EXIT dispatch table.
616	*/
617	static const struct CLANG11NOTHROWWEIRDNESS { PFNVMXEXITHANDLER pfn; } g_aVMExitHandlers[VMX_EXIT_MAX + 1] =
618	{
619	/* 0 VMX_EXIT_XCPT_OR_NMI */ { vmxHCExitXcptOrNmi },
620	/* 1 VMX_EXIT_EXT_INT */ { vmxHCExitExtInt },
621	/* 2 VMX_EXIT_TRIPLE_FAULT */ { vmxHCExitTripleFault },
622	/* 3 VMX_EXIT_INIT_SIGNAL */ { vmxHCExitErrUnexpected },
623	/* 4 VMX_EXIT_SIPI */ { vmxHCExitErrUnexpected },
624	/* 5 VMX_EXIT_IO_SMI */ { vmxHCExitErrUnexpected },
625	/* 6 VMX_EXIT_SMI */ { vmxHCExitErrUnexpected },
626	/* 7 VMX_EXIT_INT_WINDOW */ { vmxHCExitIntWindow },
627	/* 8 VMX_EXIT_NMI_WINDOW */ { vmxHCExitNmiWindow },
628	/* 9 VMX_EXIT_TASK_SWITCH */ { vmxHCExitTaskSwitch },
629	/* 10 VMX_EXIT_CPUID */ { vmxHCExitCpuid },
630	/* 11 VMX_EXIT_GETSEC */ { vmxHCExitGetsec },
631	/* 12 VMX_EXIT_HLT */ { vmxHCExitHlt },
632	/* 13 VMX_EXIT_INVD */ { vmxHCExitInvd },
633	/* 14 VMX_EXIT_INVLPG */ { vmxHCExitInvlpg },
634	/* 15 VMX_EXIT_RDPMC */ { vmxHCExitRdpmc },
635	/* 16 VMX_EXIT_RDTSC */ { vmxHCExitRdtsc },
636	/* 17 VMX_EXIT_RSM */ { vmxHCExitErrUnexpected },
637	/* 18 VMX_EXIT_VMCALL */ { vmxHCExitVmcall },
638	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
639	/* 19 VMX_EXIT_VMCLEAR */ { vmxHCExitVmclear },
640	/* 20 VMX_EXIT_VMLAUNCH */ { vmxHCExitVmlaunch },
641	/* 21 VMX_EXIT_VMPTRLD */ { vmxHCExitVmptrld },
642	/* 22 VMX_EXIT_VMPTRST */ { vmxHCExitVmptrst },
643	/* 23 VMX_EXIT_VMREAD */ { vmxHCExitVmread },
644	/* 24 VMX_EXIT_VMRESUME */ { vmxHCExitVmresume },
645	/* 25 VMX_EXIT_VMWRITE */ { vmxHCExitVmwrite },
646	/* 26 VMX_EXIT_VMXOFF */ { vmxHCExitVmxoff },
647	/* 27 VMX_EXIT_VMXON */ { vmxHCExitVmxon },
648	#else
649	/* 19 VMX_EXIT_VMCLEAR */ { vmxHCExitSetPendingXcptUD },
650	/* 20 VMX_EXIT_VMLAUNCH */ { vmxHCExitSetPendingXcptUD },
651	/* 21 VMX_EXIT_VMPTRLD */ { vmxHCExitSetPendingXcptUD },
652	/* 22 VMX_EXIT_VMPTRST */ { vmxHCExitSetPendingXcptUD },
653	/* 23 VMX_EXIT_VMREAD */ { vmxHCExitSetPendingXcptUD },
654	/* 24 VMX_EXIT_VMRESUME */ { vmxHCExitSetPendingXcptUD },
655	/* 25 VMX_EXIT_VMWRITE */ { vmxHCExitSetPendingXcptUD },
656	/* 26 VMX_EXIT_VMXOFF */ { vmxHCExitSetPendingXcptUD },
657	/* 27 VMX_EXIT_VMXON */ { vmxHCExitSetPendingXcptUD },
658	#endif
659	/* 28 VMX_EXIT_MOV_CRX */ { vmxHCExitMovCRx },
660	/* 29 VMX_EXIT_MOV_DRX */ { vmxHCExitMovDRx },
661	/* 30 VMX_EXIT_IO_INSTR */ { vmxHCExitIoInstr },
662	/* 31 VMX_EXIT_RDMSR */ { vmxHCExitRdmsr },
663	/* 32 VMX_EXIT_WRMSR */ { vmxHCExitWrmsr },
664	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ { vmxHCExitErrInvalidGuestState },
665	/* 34 VMX_EXIT_ERR_MSR_LOAD */ { vmxHCExitErrUnexpected },
666	/* 35 UNDEFINED */ { vmxHCExitErrUnexpected },
667	/* 36 VMX_EXIT_MWAIT */ { vmxHCExitMwait },
668	/* 37 VMX_EXIT_MTF */ { vmxHCExitMtf },
669	/* 38 UNDEFINED */ { vmxHCExitErrUnexpected },
670	/* 39 VMX_EXIT_MONITOR */ { vmxHCExitMonitor },
671	/* 40 VMX_EXIT_PAUSE */ { vmxHCExitPause },
672	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ { vmxHCExitErrUnexpected },
673	/* 42 UNDEFINED */ { vmxHCExitErrUnexpected },
674	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ { vmxHCExitTprBelowThreshold },
675	/* 44 VMX_EXIT_APIC_ACCESS */ { vmxHCExitApicAccess },
676	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ { vmxHCExitErrUnexpected },
677	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ { vmxHCExitErrUnexpected },
678	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ { vmxHCExitErrUnexpected },
679	/* 48 VMX_EXIT_EPT_VIOLATION */ { vmxHCExitEptViolation },
680	/* 49 VMX_EXIT_EPT_MISCONFIG */ { vmxHCExitEptMisconfig },
681	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
682	/* 50 VMX_EXIT_INVEPT */ { vmxHCExitInvept },
683	#else
684	/* 50 VMX_EXIT_INVEPT */ { vmxHCExitSetPendingXcptUD },
685	#endif
686	/* 51 VMX_EXIT_RDTSCP */ { vmxHCExitRdtscp },
687	/* 52 VMX_EXIT_PREEMPT_TIMER */ { vmxHCExitPreemptTimer },
688	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
689	/* 53 VMX_EXIT_INVVPID */ { vmxHCExitInvvpid },
690	#else
691	/* 53 VMX_EXIT_INVVPID */ { vmxHCExitSetPendingXcptUD },
692	#endif
693	/* 54 VMX_EXIT_WBINVD */ { vmxHCExitWbinvd },
694	/* 55 VMX_EXIT_XSETBV */ { vmxHCExitXsetbv },
695	/* 56 VMX_EXIT_APIC_WRITE */ { vmxHCExitErrUnexpected },
696	/* 57 VMX_EXIT_RDRAND */ { vmxHCExitErrUnexpected },
697	/* 58 VMX_EXIT_INVPCID */ { vmxHCExitInvpcid },
698	/* 59 VMX_EXIT_VMFUNC */ { vmxHCExitErrUnexpected },
699	/* 60 VMX_EXIT_ENCLS */ { vmxHCExitErrUnexpected },
700	/* 61 VMX_EXIT_RDSEED */ { vmxHCExitErrUnexpected },
701	/* 62 VMX_EXIT_PML_FULL */ { vmxHCExitErrUnexpected },
702	/* 63 VMX_EXIT_XSAVES */ { vmxHCExitErrUnexpected },
703	/* 64 VMX_EXIT_XRSTORS */ { vmxHCExitErrUnexpected },
704	/* 65 UNDEFINED */ { vmxHCExitErrUnexpected },
705	/* 66 VMX_EXIT_SPP_EVENT */ { vmxHCExitErrUnexpected },
706	/* 67 VMX_EXIT_UMWAIT */ { vmxHCExitErrUnexpected },
707	/* 68 VMX_EXIT_TPAUSE */ { vmxHCExitErrUnexpected },
708	/* 69 VMX_EXIT_LOADIWKEY */ { vmxHCExitErrUnexpected },
709	};
710	#endif /* HMVMX_USE_FUNCTION_TABLE */
711
712	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
713	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
714	{
715	/* 0 */ "(Not Used)",
716	/* 1 */ "VMCALL executed in VMX root operation.",
717	/* 2 */ "VMCLEAR with invalid physical address.",
718	/* 3 */ "VMCLEAR with VMXON pointer.",
719	/* 4 */ "VMLAUNCH with non-clear VMCS.",
720	/* 5 */ "VMRESUME with non-launched VMCS.",
721	/* 6 */ "VMRESUME after VMXOFF",
722	/* 7 */ "VM-entry with invalid control fields.",
723	/* 8 */ "VM-entry with invalid host state fields.",
724	/* 9 */ "VMPTRLD with invalid physical address.",
725	/* 10 */ "VMPTRLD with VMXON pointer.",
726	/* 11 */ "VMPTRLD with incorrect revision identifier.",
727	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
728	/* 13 */ "VMWRITE to read-only VMCS component.",
729	/* 14 */ "(Not Used)",
730	/* 15 */ "VMXON executed in VMX root operation.",
731	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
732	/* 17 */ "VM-entry with non-launched executing VMCS.",
733	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
734	/* 19 */ "VMCALL with non-clear VMCS.",
735	/* 20 */ "VMCALL with invalid VM-exit control fields.",
736	/* 21 */ "(Not Used)",
737	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
738	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
739	/* 24 */ "VMCALL with invalid SMM-monitor features.",
740	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
741	/* 26 */ "VM-entry with events blocked by MOV SS.",
742	/* 27 */ "(Not Used)",
743	/* 28 */ "Invalid operand to INVEPT/INVVPID."
744	};
745	#endif /* VBOX_STRICT && LOG_ENABLED */
746
747
748	/**
749	* Gets the CR0 guest/host mask.
750	*
751	* These bits typically does not change through the lifetime of a VM. Any bit set in
752	* this mask is owned by the host/hypervisor and would cause a VM-exit when modified
753	* by the guest.
754	*
755	* @returns The CR0 guest/host mask.
756	* @param pVCpu The cross context virtual CPU structure.
757	*/
758	static uint64_t vmxHCGetFixedCr0Mask(PCVMCPUCC pVCpu)
759	{
760	/*
761	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
762	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
763	*
764	* Furthermore, modifications to any bits that are reserved/unspecified currently
765	* by the Intel spec. must also cause a VM-exit. This prevents unpredictable behavior
766	* when future CPUs specify and use currently reserved/unspecified bits.
767	*/
768	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
769	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
770	* and @bugref{6944}. */
771	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
772	return ( X86_CR0_PE
773	\| X86_CR0_NE
774	\| (VM_IS_VMX_NESTED_PAGING(pVM) ? 0 : X86_CR0_WP)
775	\| X86_CR0_PG
776	\| VMX_EXIT_HOST_CR0_IGNORE_MASK);
777	}
778
779
780	/**
781	* Gets the CR4 guest/host mask.
782	*
783	* These bits typically does not change through the lifetime of a VM. Any bit set in
784	* this mask is owned by the host/hypervisor and would cause a VM-exit when modified
785	* by the guest.
786	*
787	* @returns The CR4 guest/host mask.
788	* @param pVCpu The cross context virtual CPU structure.
789	*/
790	static uint64_t vmxHCGetFixedCr4Mask(PCVMCPUCC pVCpu)
791	{
792	/*
793	* We construct a mask of all CR4 bits that the guest can modify without causing
794	* a VM-exit. Then invert this mask to obtain all CR4 bits that should cause
795	* a VM-exit when the guest attempts to modify them when executing using
796	* hardware-assisted VMX.
797	*
798	* When a feature is not exposed to the guest (and may be present on the host),
799	* we want to intercept guest modifications to the bit so we can emulate proper
800	* behavior (e.g., #GP).
801	*
802	* Furthermore, only modifications to those bits that don't require immediate
803	* emulation is allowed. For e.g., PCIDE is excluded because the behavior
804	* depends on CR3 which might not always be the guest value while executing
805	* using hardware-assisted VMX.
806	*/
807	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
808	bool fFsGsBase = pVM->cpum.ro.GuestFeatures.fFsGsBase;
809	#ifdef IN_NEM_DARWIN
810	bool fXSaveRstor = pVM->cpum.ro.GuestFeatures.fXSaveRstor;
811	#endif
812	bool fFxSaveRstor = pVM->cpum.ro.GuestFeatures.fFxSaveRstor;
813
814	/*
815	* Paranoia.
816	* Ensure features exposed to the guest are present on the host.
817	*/
818	AssertStmt(!fFsGsBase \|\| g_CpumHostFeatures.s.fFsGsBase, fFsGsBase = 0);
819	#ifdef IN_NEM_DARWIN
820	AssertStmt(!fXSaveRstor \|\| g_CpumHostFeatures.s.fXSaveRstor, fXSaveRstor = 0);
821	#endif
822	AssertStmt(!fFxSaveRstor \|\| g_CpumHostFeatures.s.fFxSaveRstor, fFxSaveRstor = 0);
823
824	uint64_t const fGstMask = X86_CR4_PVI
825	\| X86_CR4_TSD
826	\| X86_CR4_DE
827	\| X86_CR4_MCE
828	\| X86_CR4_PCE
829	\| X86_CR4_OSXMMEEXCPT
830	\| (fFsGsBase ? X86_CR4_FSGSBASE : 0)
831	#ifdef IN_NEM_DARWIN /* On native VT-x setting OSXSAVE must exit as we need to load guest XCR0 (see
832	fLoadSaveGuestXcr0). These exits are not needed on Darwin as that's not our problem. */
833	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
834	#endif
835	\| (fFxSaveRstor ? X86_CR4_OSFXSR : 0);
836	return ~fGstMask;
837	}
838
839
840	/**
841	* Adds one or more exceptions to the exception bitmap and commits it to the current
842	* VMCS.
843	*
844	* @param pVCpu The cross context virtual CPU structure.
845	* @param pVmxTransient The VMX-transient structure.
846	* @param uXcptMask The exception(s) to add.
847	*/
848	static void vmxHCAddXcptInterceptMask(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
849	{
850	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
851	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
852	if ((uXcptBitmap & uXcptMask) != uXcptMask)
853	{
854	uXcptBitmap \|= uXcptMask;
855	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
856	AssertRC(rc);
857	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
858	}
859	}
860
861
862	/**
863	* Adds an exception to the exception bitmap and commits it to the current VMCS.
864	*
865	* @param pVCpu The cross context virtual CPU structure.
866	* @param pVmxTransient The VMX-transient structure.
867	* @param uXcpt The exception to add.
868	*/
869	static void vmxHCAddXcptIntercept(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
870	{
871	Assert(uXcpt <= X86_XCPT_LAST);
872	vmxHCAddXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT_32(uXcpt));
873	}
874
875
876	/**
877	* Remove one or more exceptions from the exception bitmap and commits it to the
878	* current VMCS.
879	*
880	* This takes care of not removing the exception intercept if a nested-guest
881	* requires the exception to be intercepted.
882	*
883	* @returns VBox status code.
884	* @param pVCpu The cross context virtual CPU structure.
885	* @param pVmxTransient The VMX-transient structure.
886	* @param uXcptMask The exception(s) to remove.
887	*/
888	static int vmxHCRemoveXcptInterceptMask(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
889	{
890	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
891	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
892	if (u32XcptBitmap & uXcptMask)
893	{
894	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
895	if (!pVmxTransient->fIsNestedGuest)
896	{ /* likely */ }
897	else
898	uXcptMask &= ~pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32XcptBitmap;
899	#endif
900	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
901	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
902	\| RT_BIT(X86_XCPT_DE)
903	\| RT_BIT(X86_XCPT_NM)
904	\| RT_BIT(X86_XCPT_TS)
905	\| RT_BIT(X86_XCPT_UD)
906	\| RT_BIT(X86_XCPT_NP)
907	\| RT_BIT(X86_XCPT_SS)
908	\| RT_BIT(X86_XCPT_GP)
909	\| RT_BIT(X86_XCPT_PF)
910	\| RT_BIT(X86_XCPT_MF));
911	#elif defined(HMVMX_ALWAYS_TRAP_PF)
912	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
913	#endif
914	if (uXcptMask)
915	{
916	/* Validate we are not removing any essential exception intercepts. */
917	#ifndef IN_NEM_DARWIN
918	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF)));
919	#else
920	Assert(!(uXcptMask & RT_BIT(X86_XCPT_PF)));
921	#endif
922	NOREF(pVCpu);
923	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
924	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
925
926	/* Remove it from the exception bitmap. */
927	u32XcptBitmap &= ~uXcptMask;
928
929	/* Commit and update the cache if necessary. */
930	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
931	{
932	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
933	AssertRC(rc);
934	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
935	}
936	}
937	}
938	return VINF_SUCCESS;
939	}
940
941
942	/**
943	* Remove an exceptions from the exception bitmap and commits it to the current
944	* VMCS.
945	*
946	* @returns VBox status code.
947	* @param pVCpu The cross context virtual CPU structure.
948	* @param pVmxTransient The VMX-transient structure.
949	* @param uXcpt The exception to remove.
950	*/
951	static int vmxHCRemoveXcptIntercept(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
952	{
953	return vmxHCRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
954	}
955
956	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
957
958	/**
959	* Loads the shadow VMCS specified by the VMCS info. object.
960	*
961	* @returns VBox status code.
962	* @param pVmcsInfo The VMCS info. object.
963	*
964	* @remarks Can be called with interrupts disabled.
965	*/
966	static int vmxHCLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo)
967	{
968	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
969	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
970
971	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs);
972	if (RT_SUCCESS(rc))
973	pVmcsInfo->fShadowVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
974	return rc;
975	}
976
977
978	/**
979	* Clears the shadow VMCS specified by the VMCS info. object.
980	*
981	* @returns VBox status code.
982	* @param pVmcsInfo The VMCS info. object.
983	*
984	* @remarks Can be called with interrupts disabled.
985	*/
986	static int vmxHCClearShadowVmcs(PVMXVMCSINFO pVmcsInfo)
987	{
988	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
989	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
990
991	int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs);
992	if (RT_SUCCESS(rc))
993	pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
994	return rc;
995	}
996
997
998	/**
999	* Switches from and to the specified VMCSes.
1000	*
1001	* @returns VBox status code.
1002	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
1003	* @param pVmcsInfoTo The VMCS info. object we are switching to.
1004	*
1005	* @remarks Called with interrupts disabled.
1006	*/
1007	static int vmxHCSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
1008	{
1009	/*
1010	* Clear the VMCS we are switching out if it has not already been cleared.
1011	* This will sync any CPU internal data back to the VMCS.
1012	*/
1013	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1014	{
1015	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
1016	if (RT_SUCCESS(rc))
1017	{
1018	/*
1019	* The shadow VMCS, if any, would not be active at this point since we
1020	* would have cleared it while importing the virtual hardware-virtualization
1021	* state as part the VMLAUNCH/VMRESUME VM-exit. Hence, there's no need to
1022	* clear the shadow VMCS here, just assert for safety.
1023	*/
1024	Assert(!pVmcsInfoFrom->pvShadowVmcs \|\| pVmcsInfoFrom->fShadowVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR);
1025	}
1026	else
1027	return rc;
1028	}
1029
1030	/*
1031	* Clear the VMCS we are switching to if it has not already been cleared.
1032	* This will initialize the VMCS launch state to "clear" required for loading it.
1033	*
1034	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
1035	*/
1036	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1037	{
1038	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
1039	if (RT_SUCCESS(rc))
1040	{ /* likely */ }
1041	else
1042	return rc;
1043	}
1044
1045	/*
1046	* Finally, load the VMCS we are switching to.
1047	*/
1048	return hmR0VmxLoadVmcs(pVmcsInfoTo);
1049	}
1050
1051
1052	/**
1053	* Switches between the guest VMCS and the nested-guest VMCS as specified by the
1054	* caller.
1055	*
1056	* @returns VBox status code.
1057	* @param pVCpu The cross context virtual CPU structure.
1058	* @param fSwitchToNstGstVmcs Whether to switch to the nested-guest VMCS (pass
1059	* true) or guest VMCS (pass false).
1060	*/
1061	static int vmxHCSwitchToGstOrNstGstVmcs(PVMCPUCC pVCpu, bool fSwitchToNstGstVmcs)
1062	{
1063	/* Ensure we have synced everything from the guest-CPU context to the VMCS before switching. */
1064	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
1065
1066	PVMXVMCSINFO pVmcsInfoFrom;
1067	PVMXVMCSINFO pVmcsInfoTo;
1068	if (fSwitchToNstGstVmcs)
1069	{
1070	pVmcsInfoFrom = &pVCpu->hmr0.s.vmx.VmcsInfo;
1071	pVmcsInfoTo = &pVCpu->hmr0.s.vmx.VmcsInfoNstGst;
1072	}
1073	else
1074	{
1075	pVmcsInfoFrom = &pVCpu->hmr0.s.vmx.VmcsInfoNstGst;
1076	pVmcsInfoTo = &pVCpu->hmr0.s.vmx.VmcsInfo;
1077	}
1078
1079	/*
1080	* Disable interrupts to prevent being preempted while we switch the current VMCS as the
1081	* preemption hook code path acquires the current VMCS.
1082	*/
1083	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1084
1085	int rc = vmxHCSwitchVmcs(pVmcsInfoFrom, pVmcsInfoTo);
1086	if (RT_SUCCESS(rc))
1087	{
1088	pVCpu->hmr0.s.vmx.fSwitchedToNstGstVmcs = fSwitchToNstGstVmcs;
1089	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcsCopyForRing3 = fSwitchToNstGstVmcs;
1090
1091	/*
1092	* If we are switching to a VMCS that was executed on a different host CPU or was
1093	* never executed before, flag that we need to export the host state before executing
1094	* guest/nested-guest code using hardware-assisted VMX.
1095	*
1096	* This could probably be done in a preemptible context since the preemption hook
1097	* will flag the necessary change in host context. However, since preemption is
1098	* already disabled and to avoid making assumptions about host specific code in
1099	* RTMpCpuId when called with preemption enabled, we'll do this while preemption is
1100	* disabled.
1101	*/
1102	if (pVmcsInfoTo->idHostCpuState == RTMpCpuId())
1103	{ /* likely */ }
1104	else
1105	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE);
1106
1107	ASMSetFlags(fEFlags);
1108
1109	/*
1110	* We use a different VM-exit MSR-store areas for the guest and nested-guest. Hence,
1111	* flag that we need to update the host MSR values there. Even if we decide in the
1112	* future to share the VM-exit MSR-store area page between the guest and nested-guest,
1113	* if its content differs, we would have to update the host MSRs anyway.
1114	*/
1115	pVCpu->hmr0.s.vmx.fUpdatedHostAutoMsrs = false;
1116	}
1117	else
1118	ASMSetFlags(fEFlags);
1119	return rc;
1120	}
1121
1122	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
1123	#ifdef VBOX_STRICT
1124
1125	/**
1126	* Reads the VM-entry interruption-information field from the VMCS into the VMX
1127	* transient structure.
1128	*
1129	* @param pVCpu The cross context virtual CPU structure.
1130	* @param pVmxTransient The VMX-transient structure.
1131	*/
1132	DECLINLINE(void) vmxHCReadEntryIntInfoVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1133	{
1134	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
1135	AssertRC(rc);
1136	}
1137
1138
1139	/**
1140	* Reads the VM-entry exception error code field from the VMCS into
1141	* the VMX transient structure.
1142	*
1143	* @param pVCpu The cross context virtual CPU structure.
1144	* @param pVmxTransient The VMX-transient structure.
1145	*/
1146	DECLINLINE(void) vmxHCReadEntryXcptErrorCodeVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1147	{
1148	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1149	AssertRC(rc);
1150	}
1151
1152
1153	/**
1154	* Reads the VM-entry exception error code field from the VMCS into
1155	* the VMX transient structure.
1156	*
1157	* @param pVCpu The cross context virtual CPU structure.
1158	* @param pVmxTransient The VMX-transient structure.
1159	*/
1160	DECLINLINE(void) vmxHCReadEntryInstrLenVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1161	{
1162	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1163	AssertRC(rc);
1164	}
1165
1166	#endif /* VBOX_STRICT */
1167
1168
1169	/**
1170	* Reads VMCS fields into the VMXTRANSIENT structure, slow path version.
1171	*
1172	* Don't call directly unless the it's likely that some or all of the fields
1173	* given in @a a_fReadMask have already been read.
1174	*
1175	* @tparam a_fReadMask The fields to read.
1176	* @param pVCpu The cross context virtual CPU structure.
1177	* @param pVmxTransient The VMX-transient structure.
1178	*/
1179	template<uint32_t const a_fReadMask>
1180	static void vmxHCReadToTransientSlow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1181	{
1182	AssertCompile((a_fReadMask & ~( HMVMX_READ_EXIT_QUALIFICATION
1183	\| HMVMX_READ_EXIT_INSTR_LEN
1184	\| HMVMX_READ_EXIT_INSTR_INFO
1185	\| HMVMX_READ_IDT_VECTORING_INFO
1186	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
1187	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
1188	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
1189	\| HMVMX_READ_GUEST_LINEAR_ADDR
1190	\| HMVMX_READ_GUEST_PHYSICAL_ADDR
1191	\| HMVMX_READ_GUEST_PENDING_DBG_XCPTS
1192	)) == 0);
1193
1194	if ((pVmxTransient->fVmcsFieldsRead & a_fReadMask) != a_fReadMask)
1195	{
1196	uint32_t const fVmcsFieldsRead = pVmxTransient->fVmcsFieldsRead;
1197
1198	if ( (a_fReadMask & HMVMX_READ_EXIT_QUALIFICATION)
1199	&& !(fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1200	{
1201	int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1202	AssertRC(rc);
1203	}
1204	if ( (a_fReadMask & HMVMX_READ_EXIT_INSTR_LEN)
1205	&& !(fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1206	{
1207	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1208	AssertRC(rc);
1209	}
1210	if ( (a_fReadMask & HMVMX_READ_EXIT_INSTR_INFO)
1211	&& !(fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1212	{
1213	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1214	AssertRC(rc);
1215	}
1216	if ( (a_fReadMask & HMVMX_READ_IDT_VECTORING_INFO)
1217	&& !(fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1218	{
1219	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1220	AssertRC(rc);
1221	}
1222	if ( (a_fReadMask & HMVMX_READ_IDT_VECTORING_ERROR_CODE)
1223	&& !(fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1224	{
1225	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1226	AssertRC(rc);
1227	}
1228	if ( (a_fReadMask & HMVMX_READ_EXIT_INTERRUPTION_INFO)
1229	&& !(fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1230	{
1231	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1232	AssertRC(rc);
1233	}
1234	if ( (a_fReadMask & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE)
1235	&& !(fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1236	{
1237	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1238	AssertRC(rc);
1239	}
1240	if ( (a_fReadMask & HMVMX_READ_GUEST_LINEAR_ADDR)
1241	&& !(fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1242	{
1243	int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1244	AssertRC(rc);
1245	}
1246	if ( (a_fReadMask & HMVMX_READ_GUEST_PHYSICAL_ADDR)
1247	&& !(fVmcsFieldsRead & HMVMX_READ_GUEST_PHYSICAL_ADDR))
1248	{
1249	int const rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1250	AssertRC(rc);
1251	}
1252	if ( (a_fReadMask & HMVMX_READ_GUEST_PENDING_DBG_XCPTS)
1253	&& !(fVmcsFieldsRead & HMVMX_READ_GUEST_PENDING_DBG_XCPTS))
1254	{
1255	int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &pVmxTransient->uGuestPendingDbgXcpts);
1256	AssertRC(rc);
1257	}
1258
1259	pVmxTransient->fVmcsFieldsRead \|= a_fReadMask;
1260	}
1261	}
1262
1263
1264	/**
1265	* Reads VMCS fields into the VMXTRANSIENT structure.
1266	*
1267	* This optimizes for the case where none of @a a_fReadMask has been read yet,
1268	* generating an optimized read sequences w/o any conditionals between in
1269	* non-strict builds.
1270	*
1271	* @tparam a_fReadMask The fields to read. One or more of the
1272	* HMVMX_READ_XXX fields ORed together.
1273	* @param pVCpu The cross context virtual CPU structure.
1274	* @param pVmxTransient The VMX-transient structure.
1275	*/
1276	template<uint32_t const a_fReadMask>
1277	DECLINLINE(void) vmxHCReadToTransient(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1278	{
1279	AssertCompile((a_fReadMask & ~( HMVMX_READ_EXIT_QUALIFICATION
1280	\| HMVMX_READ_EXIT_INSTR_LEN
1281	\| HMVMX_READ_EXIT_INSTR_INFO
1282	\| HMVMX_READ_IDT_VECTORING_INFO
1283	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
1284	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
1285	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
1286	\| HMVMX_READ_GUEST_LINEAR_ADDR
1287	\| HMVMX_READ_GUEST_PHYSICAL_ADDR
1288	\| HMVMX_READ_GUEST_PENDING_DBG_XCPTS
1289	)) == 0);
1290
1291	if (RT_LIKELY(!(pVmxTransient->fVmcsFieldsRead & a_fReadMask)))
1292	{
1293	if (a_fReadMask & HMVMX_READ_EXIT_QUALIFICATION)
1294	{
1295	int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1296	AssertRC(rc);
1297	}
1298	if (a_fReadMask & HMVMX_READ_EXIT_INSTR_LEN)
1299	{
1300	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1301	AssertRC(rc);
1302	}
1303	if (a_fReadMask & HMVMX_READ_EXIT_INSTR_INFO)
1304	{
1305	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1306	AssertRC(rc);
1307	}
1308	if (a_fReadMask & HMVMX_READ_IDT_VECTORING_INFO)
1309	{
1310	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1311	AssertRC(rc);
1312	}
1313	if (a_fReadMask & HMVMX_READ_IDT_VECTORING_ERROR_CODE)
1314	{
1315	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1316	AssertRC(rc);
1317	}
1318	if (a_fReadMask & HMVMX_READ_EXIT_INTERRUPTION_INFO)
1319	{
1320	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1321	AssertRC(rc);
1322	}
1323	if (a_fReadMask & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE)
1324	{
1325	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1326	AssertRC(rc);
1327	}
1328	if (a_fReadMask & HMVMX_READ_GUEST_LINEAR_ADDR)
1329	{
1330	int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1331	AssertRC(rc);
1332	}
1333	if (a_fReadMask & HMVMX_READ_GUEST_PHYSICAL_ADDR)
1334	{
1335	int const rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1336	AssertRC(rc);
1337	}
1338	if (a_fReadMask & HMVMX_READ_GUEST_PENDING_DBG_XCPTS)
1339	{
1340	int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &pVmxTransient->uGuestPendingDbgXcpts);
1341	AssertRC(rc);
1342	}
1343
1344	pVmxTransient->fVmcsFieldsRead \|= a_fReadMask;
1345	}
1346	else
1347	vmxHCReadToTransientSlow<a_fReadMask>(pVCpu, pVmxTransient);
1348	}
1349
1350
1351	#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
1352	/**
1353	* Reads all relevant read-only VMCS fields into the VMX transient structure.
1354	*
1355	* @param pVCpu The cross context virtual CPU structure.
1356	* @param pVmxTransient The VMX-transient structure.
1357	*/
1358	static void vmxHCReadAllRoFieldsVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1359	{
1360	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1361	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1362	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1363	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1364	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1365	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1366	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1367	rc \|= VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1368	rc \|= VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1369	AssertRC(rc);
1370	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION
1371	\| HMVMX_READ_EXIT_INSTR_LEN
1372	\| HMVMX_READ_EXIT_INSTR_INFO
1373	\| HMVMX_READ_IDT_VECTORING_INFO
1374	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
1375	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
1376	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
1377	\| HMVMX_READ_GUEST_LINEAR_ADDR
1378	\| HMVMX_READ_GUEST_PHYSICAL_ADDR;
1379	}
1380	#endif
1381
1382	/**
1383	* Verifies that our cached values of the VMCS fields are all consistent with
1384	* what's actually present in the VMCS.
1385	*
1386	* @returns VBox status code.
1387	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
1388	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
1389	* VMCS content. HMCPU error-field is
1390	* updated, see VMX_VCI_XXX.
1391	* @param pVCpu The cross context virtual CPU structure.
1392	* @param pVmcsInfo The VMCS info. object.
1393	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1394	*/
1395	static int vmxHCCheckCachedVmcsCtls(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1396	{
1397	const char * const pcszVmcs = fIsNstGstVmcs ? "Nested-guest VMCS" : "VMCS";
1398
1399	uint32_t u32Val;
1400	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY, &u32Val);
1401	AssertRC(rc);
1402	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
1403	("%s entry controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32EntryCtls, u32Val),
1404	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_ENTRY,
1405	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1406
1407	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_EXIT, &u32Val);
1408	AssertRC(rc);
1409	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
1410	("%s exit controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ExitCtls, u32Val),
1411	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_EXIT,
1412	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1413
1414	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1415	AssertRC(rc);
1416	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
1417	("%s pin controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32PinCtls, u32Val),
1418	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PIN_EXEC,
1419	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1420
1421	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1422	AssertRC(rc);
1423	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
1424	("%s proc controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls, u32Val),
1425	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PROC_EXEC,
1426	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1427
1428	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
1429	{
1430	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1431	AssertRC(rc);
1432	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
1433	("%s proc2 controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls2, u32Val),
1434	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
1435	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1436	}
1437
1438	uint64_t u64Val;
1439	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TERTIARY_CTLS)
1440	{
1441	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_CTRL_PROC_EXEC3_FULL, &u64Val);
1442	AssertRC(rc);
1443	AssertMsgReturnStmt(pVmcsInfo->u64ProcCtls3 == u64Val,
1444	("%s proc3 controls mismatch: Cache=%#RX32 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64ProcCtls3, u64Val),
1445	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PROC_EXEC3,
1446	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1447	}
1448
1449	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
1450	AssertRC(rc);
1451	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
1452	("%s exception bitmap mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32XcptBitmap, u32Val),
1453	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
1454	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1455
1456	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
1457	AssertRC(rc);
1458	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
1459	("%s TSC offset mismatch: Cache=%#RX64 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64TscOffset, u64Val),
1460	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
1461	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1462
1463	NOREF(pcszVmcs);
1464	return VINF_SUCCESS;
1465	}
1466
1467
1468	/**
1469	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
1470	* VMCS.
1471	*
1472	* This is typically required when the guest changes paging mode.
1473	*
1474	* @returns VBox status code.
1475	* @param pVCpu The cross context virtual CPU structure.
1476	* @param pVmxTransient The VMX-transient structure.
1477	*
1478	* @remarks Requires EFER.
1479	* @remarks No-long-jump zone!!!
1480	*/
1481	static int vmxHCExportGuestEntryExitCtls(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
1482	{
1483	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
1484	{
1485	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1486	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1487
1488	/*
1489	* VM-entry controls.
1490	*/
1491	{
1492	uint32_t fVal = g_HmMsrs.u.vmx.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
1493	uint32_t const fZap = g_HmMsrs.u.vmx.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1494
1495	/*
1496	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
1497	* The first VT-x capable CPUs only supported the 1-setting of this bit.
1498	*
1499	* For nested-guests, this is a mandatory VM-entry control. It's also
1500	* required because we do not want to leak host bits to the nested-guest.
1501	*/
1502	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
1503
1504	/*
1505	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
1506	*
1507	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
1508	* required to get the nested-guest working with hardware-assisted VMX execution.
1509	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested hypervisor
1510	* can skip intercepting changes to the EFER MSR. This is why it needs to be done
1511	* here rather than while merging the guest VMCS controls.
1512	*/
1513	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
1514	{
1515	Assert(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME);
1516	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
1517	}
1518	else
1519	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
1520
1521	/*
1522	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
1523	*
1524	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
1525	* regardless of whether the nested-guest VMCS specifies it because we are free to
1526	* load whatever MSRs we require and we do not need to modify the guest visible copy
1527	* of the VM-entry MSR load area.
1528	*/
1529	if ( g_fHmVmxSupportsVmcsEfer
1530	#ifndef IN_NEM_DARWIN
1531	&& hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient)
1532	#endif
1533	)
1534	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
1535	else
1536	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
1537
1538	/*
1539	* The following should -not- be set (since we're not in SMM mode):
1540	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
1541	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
1542	*/
1543
1544	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
1545	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
1546
1547	if ((fVal & fZap) == fVal)
1548	{ /* likely */ }
1549	else
1550	{
1551	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
1552	g_HmMsrs.u.vmx.EntryCtls.n.allowed0, fVal, fZap));
1553	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_CTRL_ENTRY;
1554	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1555	}
1556
1557	/* Commit it to the VMCS. */
1558	if (pVmcsInfo->u32EntryCtls != fVal)
1559	{
1560	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY, fVal);
1561	AssertRC(rc);
1562	pVmcsInfo->u32EntryCtls = fVal;
1563	}
1564	}
1565
1566	/*
1567	* VM-exit controls.
1568	*/
1569	{
1570	uint32_t fVal = g_HmMsrs.u.vmx.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
1571	uint32_t const fZap = g_HmMsrs.u.vmx.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1572
1573	/*
1574	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
1575	* supported the 1-setting of this bit.
1576	*
1577	* For nested-guests, we set the "save debug controls" as the converse
1578	* "load debug controls" is mandatory for nested-guests anyway.
1579	*/
1580	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
1581
1582	/*
1583	* Set the host long mode active (EFER.LMA) bit (which Intel calls
1584	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
1585	* host EFER.LMA and EFER.LME bit to this value. See assertion in
1586	* vmxHCExportHostMsrs().
1587	*
1588	* For nested-guests, we always set this bit as we do not support 32-bit
1589	* hosts.
1590	*/
1591	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
1592
1593	#ifndef IN_NEM_DARWIN
1594	/*
1595	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
1596	*
1597	* For nested-guests, we should use the "save IA32_EFER" control if we also
1598	* used the "load IA32_EFER" control while exporting VM-entry controls.
1599	*/
1600	if ( g_fHmVmxSupportsVmcsEfer
1601	&& hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
1602	{
1603	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
1604	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
1605	}
1606	#endif
1607
1608	/*
1609	* Enable saving of the VMX-preemption timer value on VM-exit.
1610	* For nested-guests, currently not exposed/used.
1611	*/
1612	/** @todo r=bird: Measure performance hit because of this vs. always rewriting
1613	* the timer value. */
1614	if (VM_IS_VMX_PREEMPT_TIMER_USED(pVM))
1615	{
1616	Assert(g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER);
1617	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
1618	}
1619
1620	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
1621	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
1622
1623	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
1624	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
1625	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
1626
1627	if ((fVal & fZap) == fVal)
1628	{ /* likely */ }
1629	else
1630	{
1631	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
1632	g_HmMsrs.u.vmx.ExitCtls.n.allowed0, fVal, fZap));
1633	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_CTRL_EXIT;
1634	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1635	}
1636
1637	/* Commit it to the VMCS. */
1638	if (pVmcsInfo->u32ExitCtls != fVal)
1639	{
1640	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXIT, fVal);
1641	AssertRC(rc);
1642	pVmcsInfo->u32ExitCtls = fVal;
1643	}
1644	}
1645
1646	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
1647	}
1648	return VINF_SUCCESS;
1649	}
1650
1651
1652	/**
1653	* Sets the TPR threshold in the VMCS.
1654	*
1655	* @param pVCpu The cross context virtual CPU structure.
1656	* @param pVmcsInfo The VMCS info. object.
1657	* @param u32TprThreshold The TPR threshold (task-priority class only).
1658	*/
1659	DECLINLINE(void) vmxHCApicSetTprThreshold(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
1660	{
1661	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
1662	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
1663	RT_NOREF(pVmcsInfo);
1664	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
1665	AssertRC(rc);
1666	}
1667
1668
1669	/**
1670	* Exports the guest APIC TPR state into the VMCS.
1671	*
1672	* @param pVCpu The cross context virtual CPU structure.
1673	* @param pVmxTransient The VMX-transient structure.
1674	*
1675	* @remarks No-long-jump zone!!!
1676	*/
1677	static void vmxHCExportGuestApicTpr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
1678	{
1679	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
1680	{
1681	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
1682
1683	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1684	if (!pVmxTransient->fIsNestedGuest)
1685	{
1686	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1687	&& APICIsEnabled(pVCpu))
1688	{
1689	/*
1690	* Setup TPR shadowing.
1691	*/
1692	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
1693	{
1694	bool fPendingIntr = false;
1695	uint8_t u8Tpr = 0;
1696	uint8_t u8PendingIntr = 0;
1697	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
1698	AssertRC(rc);
1699
1700	/*
1701	* If there are interrupts pending but masked by the TPR, instruct VT-x to
1702	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
1703	* priority of the pending interrupt so we can deliver the interrupt. If there
1704	* are no interrupts pending, set threshold to 0 to not cause any
1705	* TPR-below-threshold VM-exits.
1706	*/
1707	uint32_t u32TprThreshold = 0;
1708	if (fPendingIntr)
1709	{
1710	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
1711	(which is the Task-Priority Class). */
1712	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
1713	const uint8_t u8TprPriority = u8Tpr >> 4;
1714	if (u8PendingPriority <= u8TprPriority)
1715	u32TprThreshold = u8PendingPriority;
1716	}
1717
1718	vmxHCApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
1719	}
1720	}
1721	}
1722	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
1723	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
1724	}
1725	}
1726
1727
1728	/**
1729	* Gets the guest interruptibility-state and updates related force-flags.
1730	*
1731	* @returns Guest's interruptibility-state.
1732	* @param pVCpu The cross context virtual CPU structure.
1733	*
1734	* @remarks No-long-jump zone!!!
1735	*/
1736	static uint32_t vmxHCGetGuestIntrStateAndUpdateFFs(PVMCPUCC pVCpu)
1737	{
1738	/*
1739	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
1740	*/
1741	uint32_t fIntrState = 0;
1742	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1743	{
1744	/* If inhibition is active, RIP and RFLAGS should've been imported from the VMCS already. */
1745	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
1746
1747	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1748	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
1749	{
1750	if (pCtx->eflags.Bits.u1IF)
1751	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
1752	else
1753	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
1754	}
1755	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1756	{
1757	/*
1758	* We can clear the inhibit force flag as even if we go back to the recompiler
1759	* without executing guest code in VT-x, the flag's condition to be cleared is
1760	* met and thus the cleared state is correct.
1761	*/
1762	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1763	}
1764	}
1765
1766	/*
1767	* Check if we should inhibit NMI delivery.
1768	*/
1769	if (CPUMIsGuestNmiBlocking(pVCpu))
1770	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
1771
1772	/*
1773	* Validate.
1774	*/
1775	#ifdef VBOX_STRICT
1776	/* We don't support block-by-SMI yet.*/
1777	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI));
1778
1779	/* Block-by-STI must not be set when interrupts are disabled. */
1780	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
1781	{
1782	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
1783	Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_IF);
1784	}
1785	#endif
1786
1787	return fIntrState;
1788	}
1789
1790
1791	/**
1792	* Exports the exception intercepts required for guest execution in the VMCS.
1793	*
1794	* @param pVCpu The cross context virtual CPU structure.
1795	* @param pVmxTransient The VMX-transient structure.
1796	*
1797	* @remarks No-long-jump zone!!!
1798	*/
1799	static void vmxHCExportGuestXcptIntercepts(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
1800	{
1801	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
1802	{
1803	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
1804	if ( !pVmxTransient->fIsNestedGuest
1805	&& VCPU_2_VMXSTATE(pVCpu).fGIMTrapXcptUD)
1806	vmxHCAddXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
1807	else
1808	vmxHCRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
1809
1810	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
1811	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
1812	}
1813	}
1814
1815
1816	/**
1817	* Exports the guest's RIP into the guest-state area in the VMCS.
1818	*
1819	* @param pVCpu The cross context virtual CPU structure.
1820	*
1821	* @remarks No-long-jump zone!!!
1822	*/
1823	static void vmxHCExportGuestRip(PVMCPUCC pVCpu)
1824	{
1825	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_RIP)
1826	{
1827	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
1828
1829	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
1830	AssertRC(rc);
1831
1832	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_RIP);
1833	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
1834	}
1835	}
1836
1837
1838	/**
1839	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
1840	*
1841	* @param pVCpu The cross context virtual CPU structure.
1842	* @param pVmxTransient The VMX-transient structure.
1843	*
1844	* @remarks No-long-jump zone!!!
1845	*/
1846	static void vmxHCExportGuestRflags(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
1847	{
1848	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
1849	{
1850	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
1851
1852	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
1853	Let us assert it as such and use 32-bit VMWRITE. */
1854	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
1855	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
1856	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
1857	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
1858
1859	#ifndef IN_NEM_DARWIN
1860	/*
1861	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
1862	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
1863	* can run the real-mode guest code under Virtual 8086 mode.
1864	*/
1865	PVMXVMCSINFOSHARED pVmcsInfo = pVmxTransient->pVmcsInfo->pShared;
1866	if (pVmcsInfo->RealMode.fRealOnV86Active)
1867	{
1868	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
1869	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
1870	Assert(!pVmxTransient->fIsNestedGuest);
1871	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
1872	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
1873	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
1874	}
1875	#else
1876	RT_NOREF(pVmxTransient);
1877	#endif
1878
1879	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
1880	AssertRC(rc);
1881
1882	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
1883	Log4Func(("eflags=%#RX32\n", fEFlags.u32));
1884	}
1885	}
1886
1887
1888	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1889	/**
1890	* Copies the nested-guest VMCS to the shadow VMCS.
1891	*
1892	* @returns VBox status code.
1893	* @param pVCpu The cross context virtual CPU structure.
1894	* @param pVmcsInfo The VMCS info. object.
1895	*
1896	* @remarks No-long-jump zone!!!
1897	*/
1898	static int vmxHCCopyNstGstToShadowVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
1899	{
1900	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
1901	PCVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
1902
1903	/*
1904	* Disable interrupts so we don't get preempted while the shadow VMCS is the
1905	* current VMCS, as we may try saving guest lazy MSRs.
1906	*
1907	* Strictly speaking the lazy MSRs are not in the VMCS, but I'd rather not risk
1908	* calling the import VMCS code which is currently performing the guest MSR reads
1909	* (on 64-bit hosts) and accessing the auto-load/store MSR area on 32-bit hosts
1910	* and the rest of the VMX leave session machinery.
1911	*/
1912	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1913
1914	int rc = vmxHCLoadShadowVmcs(pVmcsInfo);
1915	if (RT_SUCCESS(rc))
1916	{
1917	/*
1918	* Copy all guest read/write VMCS fields.
1919	*
1920	* We don't check for VMWRITE failures here for performance reasons and
1921	* because they are not expected to fail, barring irrecoverable conditions
1922	* like hardware errors.
1923	*/
1924	uint32_t const cShadowVmcsFields = pVM->hmr0.s.vmx.cShadowVmcsFields;
1925	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
1926	{
1927	uint64_t u64Val;
1928	uint32_t const uVmcsField = pVM->hmr0.s.vmx.paShadowVmcsFields[i];
1929	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
1930	VMX_VMCS_WRITE_64(pVCpu, uVmcsField, u64Val);
1931	}
1932
1933	/*
1934	* If the host CPU supports writing all VMCS fields, copy the guest read-only
1935	* VMCS fields, so the guest can VMREAD them without causing a VM-exit.
1936	*/
1937	if (g_HmMsrs.u.vmx.u64Misc & VMX_MISC_VMWRITE_ALL)
1938	{
1939	uint32_t const cShadowVmcsRoFields = pVM->hmr0.s.vmx.cShadowVmcsRoFields;
1940	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
1941	{
1942	uint64_t u64Val;
1943	uint32_t const uVmcsField = pVM->hmr0.s.vmx.paShadowVmcsRoFields[i];
1944	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
1945	VMX_VMCS_WRITE_64(pVCpu, uVmcsField, u64Val);
1946	}
1947	}
1948
1949	rc = vmxHCClearShadowVmcs(pVmcsInfo);
1950	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
1951	}
1952
1953	ASMSetFlags(fEFlags);
1954	return rc;
1955	}
1956
1957
1958	/**
1959	* Copies the shadow VMCS to the nested-guest VMCS.
1960	*
1961	* @returns VBox status code.
1962	* @param pVCpu The cross context virtual CPU structure.
1963	* @param pVmcsInfo The VMCS info. object.
1964	*
1965	* @remarks Called with interrupts disabled.
1966	*/
1967	static int vmxHCCopyShadowToNstGstVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
1968	{
1969	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1970	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
1971	PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
1972
1973	int rc = vmxHCLoadShadowVmcs(pVmcsInfo);
1974	if (RT_SUCCESS(rc))
1975	{
1976	/*
1977	* Copy guest read/write fields from the shadow VMCS.
1978	* Guest read-only fields cannot be modified, so no need to copy them.
1979	*
1980	* We don't check for VMREAD failures here for performance reasons and
1981	* because they are not expected to fail, barring irrecoverable conditions
1982	* like hardware errors.
1983	*/
1984	uint32_t const cShadowVmcsFields = pVM->hmr0.s.vmx.cShadowVmcsFields;
1985	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
1986	{
1987	uint64_t u64Val;
1988	uint32_t const uVmcsField = pVM->hmr0.s.vmx.paShadowVmcsFields[i];
1989	VMX_VMCS_READ_64(pVCpu, uVmcsField, &u64Val);
1990	IEMWriteVmxVmcsField(pVmcsNstGst, uVmcsField, u64Val);
1991	}
1992
1993	rc = vmxHCClearShadowVmcs(pVmcsInfo);
1994	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
1995	}
1996	return rc;
1997	}
1998
1999
2000	/**
2001	* Enables VMCS shadowing for the given VMCS info. object.
2002	*
2003	* @param pVCpu The cross context virtual CPU structure.
2004	* @param pVmcsInfo The VMCS info. object.
2005	*
2006	* @remarks No-long-jump zone!!!
2007	*/
2008	static void vmxHCEnableVmcsShadowing(PCVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2009	{
2010	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
2011	if (!(uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING))
2012	{
2013	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
2014	uProcCtls2 \|= VMX_PROC_CTLS2_VMCS_SHADOWING;
2015	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc);
2016	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, pVmcsInfo->HCPhysShadowVmcs); AssertRC(rc);
2017	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
2018	pVmcsInfo->u64VmcsLinkPtr = pVmcsInfo->HCPhysShadowVmcs;
2019	Log4Func(("Enabled\n"));
2020	}
2021	}
2022
2023
2024	/**
2025	* Disables VMCS shadowing for the given VMCS info. object.
2026	*
2027	* @param pVCpu The cross context virtual CPU structure.
2028	* @param pVmcsInfo The VMCS info. object.
2029	*
2030	* @remarks No-long-jump zone!!!
2031	*/
2032	static void vmxHCDisableVmcsShadowing(PCVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2033	{
2034	/*
2035	* We want all VMREAD and VMWRITE instructions to cause VM-exits, so we clear the
2036	* VMCS shadowing control. However, VM-entry requires the shadow VMCS indicator bit
2037	* to match the VMCS shadowing control if the VMCS link pointer is not NIL_RTHCPHYS.
2038	* Hence, we must also reset the VMCS link pointer to ensure VM-entry does not fail.
2039	*
2040	* See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
2041	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
2042	*/
2043	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
2044	if (uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
2045	{
2046	uProcCtls2 &= ~VMX_PROC_CTLS2_VMCS_SHADOWING;
2047	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc);
2048	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS); AssertRC(rc);
2049	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
2050	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
2051	Log4Func(("Disabled\n"));
2052	}
2053	}
2054	#endif
2055
2056
2057	/**
2058	* Exports the guest CR0 control register into the guest-state area in the VMCS.
2059	*
2060	* The guest FPU state is always pre-loaded hence we don't need to bother about
2061	* sharing FPU related CR0 bits between the guest and host.
2062	*
2063	* @returns VBox status code.
2064	* @param pVCpu The cross context virtual CPU structure.
2065	* @param pVmxTransient The VMX-transient structure.
2066	*
2067	* @remarks No-long-jump zone!!!
2068	*/
2069	static int vmxHCExportGuestCR0(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
2070	{
2071	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CR0)
2072	{
2073	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2074	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2075
2076	uint64_t fSetCr0 = g_HmMsrs.u.vmx.u64Cr0Fixed0;
2077	uint64_t const fZapCr0 = g_HmMsrs.u.vmx.u64Cr0Fixed1;
2078	if (VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
2079	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
2080	else
2081	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2082
2083	if (!pVmxTransient->fIsNestedGuest)
2084	{
2085	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
2086	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
2087	uint64_t const u64ShadowCr0 = u64GuestCr0;
2088	Assert(!RT_HI_U32(u64GuestCr0));
2089
2090	/*
2091	* Setup VT-x's view of the guest CR0.
2092	*/
2093	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
2094	if (VM_IS_VMX_NESTED_PAGING(pVM))
2095	{
2096	#ifndef HMVMX_ALWAYS_INTERCEPT_CR3_ACCESS
2097	if (CPUMIsGuestPagingEnabled(pVCpu))
2098	{
2099	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
2100	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
2101	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
2102	}
2103	else
2104	{
2105	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
2106	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
2107	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
2108	}
2109
2110	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
2111	if (VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
2112	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
2113	#endif
2114	}
2115	else
2116	{
2117	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
2118	u64GuestCr0 \|= X86_CR0_WP;
2119	}
2120
2121	/*
2122	* Guest FPU bits.
2123	*
2124	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
2125	* using CR0.TS.
2126	*
2127	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
2128	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2129	*/
2130	u64GuestCr0 \|= X86_CR0_NE;
2131
2132	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
2133	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
2134
2135	/*
2136	* Update exception intercepts.
2137	*/
2138	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
2139	#ifndef IN_NEM_DARWIN
2140	if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
2141	{
2142	Assert(PDMVmmDevHeapIsEnabled(pVM));
2143	Assert(pVM->hm.s.vmx.pRealModeTSS);
2144	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2145	}
2146	else
2147	#endif
2148	{
2149	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
2150	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2151	if (fInterceptMF)
2152	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
2153	}
2154
2155	/* Additional intercepts for debugging, define these yourself explicitly. */
2156	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
2157	uXcptBitmap \|= 0
2158	\| RT_BIT(X86_XCPT_BP)
2159	\| RT_BIT(X86_XCPT_DE)
2160	\| RT_BIT(X86_XCPT_NM)
2161	\| RT_BIT(X86_XCPT_TS)
2162	\| RT_BIT(X86_XCPT_UD)
2163	\| RT_BIT(X86_XCPT_NP)
2164	\| RT_BIT(X86_XCPT_SS)
2165	\| RT_BIT(X86_XCPT_GP)
2166	\| RT_BIT(X86_XCPT_PF)
2167	\| RT_BIT(X86_XCPT_MF)
2168	;
2169	#elif defined(HMVMX_ALWAYS_TRAP_PF)
2170	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
2171	#endif
2172	if (VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv)
2173	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
2174	if (VCPU_2_VMXSTATE(pVCpu).fGCMTrapXcptDE)
2175	uXcptBitmap \|= RT_BIT(X86_XCPT_DE);
2176	Assert(VM_IS_VMX_NESTED_PAGING(pVM) \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
2177
2178	/* Apply the hardware specified CR0 fixed bits and enable caching. */
2179	u64GuestCr0 \|= fSetCr0;
2180	u64GuestCr0 &= fZapCr0;
2181	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
2182
2183	/* Commit the CR0 and related fields to the guest VMCS. */
2184	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc);
2185	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc);
2186	if (uProcCtls != pVmcsInfo->u32ProcCtls)
2187	{
2188	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
2189	AssertRC(rc);
2190	}
2191	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
2192	{
2193	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2194	AssertRC(rc);
2195	}
2196
2197	/* Update our caches. */
2198	pVmcsInfo->u32ProcCtls = uProcCtls;
2199	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
2200
2201	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
2202	}
2203	else
2204	{
2205	/*
2206	* With nested-guests, we may have extended the guest/host mask here since we
2207	* merged in the outer guest's mask. Thus, the merged mask can include more bits
2208	* (to read from the nested-guest CR0 read-shadow) than the nested hypervisor
2209	* originally supplied. We must copy those bits from the nested-guest CR0 into
2210	* the nested-guest CR0 read-shadow.
2211	*/
2212	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
2213	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
2214	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(&pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
2215	Assert(!RT_HI_U32(u64GuestCr0));
2216	Assert(u64GuestCr0 & X86_CR0_NE);
2217
2218	/* Apply the hardware specified CR0 fixed bits and enable caching. */
2219	u64GuestCr0 \|= fSetCr0;
2220	u64GuestCr0 &= fZapCr0;
2221	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
2222
2223	/* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
2224	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc);
2225	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc);
2226
2227	Log4Func(("cr0=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
2228	}
2229
2230	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CR0);
2231	}
2232
2233	return VINF_SUCCESS;
2234	}
2235
2236
2237	/**
2238	* Exports the guest control registers (CR3, CR4) into the guest-state area
2239	* in the VMCS.
2240	*
2241	* @returns VBox strict status code.
2242	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
2243	* without unrestricted guest access and the VMMDev is not presently
2244	* mapped (e.g. EFI32).
2245	*
2246	* @param pVCpu The cross context virtual CPU structure.
2247	* @param pVmxTransient The VMX-transient structure.
2248	*
2249	* @remarks No-long-jump zone!!!
2250	*/
2251	static VBOXSTRICTRC vmxHCExportGuestCR3AndCR4(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
2252	{
2253	int rc = VINF_SUCCESS;
2254	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2255
2256	/*
2257	* Guest CR2.
2258	* It's always loaded in the assembler code. Nothing to do here.
2259	*/
2260
2261	/*
2262	* Guest CR3.
2263	*/
2264	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CR3)
2265	{
2266	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
2267
2268	if (VM_IS_VMX_NESTED_PAGING(pVM))
2269	{
2270	#ifndef IN_NEM_DARWIN
2271	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2272	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
2273
2274	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2275	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
2276	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
2277	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
2278
2279	/* VMX_EPT_MEMTYPE_WB support is already checked in vmxHCSetupTaggedTlb(). */
2280	pVmcsInfo->HCPhysEPTP \|= RT_BF_MAKE(VMX_BF_EPTP_MEMTYPE, VMX_EPTP_MEMTYPE_WB)
2281	\| RT_BF_MAKE(VMX_BF_EPTP_PAGE_WALK_LENGTH, VMX_EPTP_PAGE_WALK_LENGTH_4);
2282
2283	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2284	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2285	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
2286	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
2287	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
2288	\|\| (g_HmMsrs.u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_ACCESS_DIRTY),
2289	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
2290
2291	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
2292	AssertRC(rc);
2293	#endif
2294
2295	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2296	uint64_t u64GuestCr3 = pCtx->cr3;
2297	if ( VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
2298	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2299	{
2300	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2301	if (CPUMIsGuestInPAEModeEx(pCtx))
2302	{
2303	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, pCtx->aPaePdpes[0].u); AssertRC(rc);
2304	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, pCtx->aPaePdpes[1].u); AssertRC(rc);
2305	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, pCtx->aPaePdpes[2].u); AssertRC(rc);
2306	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, pCtx->aPaePdpes[3].u); AssertRC(rc);
2307	}
2308
2309	/*
2310	* The guest's view of its CR3 is unblemished with nested paging when the
2311	* guest is using paging or we have unrestricted guest execution to handle
2312	* the guest when it's not using paging.
2313	*/
2314	}
2315	#ifndef IN_NEM_DARWIN
2316	else
2317	{
2318	/*
2319	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
2320	* thinks it accesses physical memory directly, we use our identity-mapped
2321	* page table to map guest-linear to guest-physical addresses. EPT takes care
2322	* of translating it to host-physical addresses.
2323	*/
2324	RTGCPHYS GCPhys;
2325	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2326
2327	/* We obtain it here every time as the guest could have relocated this PCI region. */
2328	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2329	if (RT_SUCCESS(rc))
2330	{ /* likely */ }
2331	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
2332	{
2333	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
2334	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
2335	}
2336	else
2337	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
2338
2339	u64GuestCr3 = GCPhys;
2340	}
2341	#endif
2342
2343	Log4Func(("guest_cr3=%#RX64 (GstN)\n", u64GuestCr3));
2344	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR3, u64GuestCr3);
2345	AssertRC(rc);
2346	}
2347	else
2348	{
2349	Assert(!pVmxTransient->fIsNestedGuest);
2350	/* Non-nested paging case, just use the hypervisor's CR3. */
2351	RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);
2352
2353	Log4Func(("guest_cr3=%#RX64 (HstN)\n", HCPhysGuestCr3));
2354	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
2355	AssertRC(rc);
2356	}
2357
2358	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CR3);
2359	}
2360
2361	/*
2362	* Guest CR4.
2363	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
2364	*/
2365	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CR4)
2366	{
2367	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2368	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2369
2370	uint64_t const fSetCr4 = g_HmMsrs.u.vmx.u64Cr4Fixed0;
2371	uint64_t const fZapCr4 = g_HmMsrs.u.vmx.u64Cr4Fixed1;
2372
2373	/*
2374	* With nested-guests, we may have extended the guest/host mask here (since we
2375	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
2376	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
2377	* the nested hypervisor originally supplied. Thus, we should, in essence, copy
2378	* those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
2379	*/
2380	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
2381	uint64_t u64GuestCr4 = pCtx->cr4;
2382	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
2383	? pCtx->cr4
2384	: CPUMGetGuestVmxMaskedCr4(pCtx, pVmcsInfo->u64Cr4Mask);
2385	Assert(!RT_HI_U32(u64GuestCr4));
2386
2387	#ifndef IN_NEM_DARWIN
2388	/*
2389	* Setup VT-x's view of the guest CR4.
2390	*
2391	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
2392	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
2393	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2394	*
2395	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2396	*/
2397	if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
2398	{
2399	Assert(pVM->hm.s.vmx.pRealModeTSS);
2400	Assert(PDMVmmDevHeapIsEnabled(pVM));
2401	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
2402	}
2403	#endif
2404
2405	if (VM_IS_VMX_NESTED_PAGING(pVM))
2406	{
2407	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2408	&& !VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
2409	{
2410	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2411	u64GuestCr4 \|= X86_CR4_PSE;
2412	/* Our identity mapping is a 32-bit page directory. */
2413	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
2414	}
2415	/* else use guest CR4.*/
2416	}
2417	else
2418	{
2419	Assert(!pVmxTransient->fIsNestedGuest);
2420
2421	/*
2422	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2423	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2424	*/
2425	switch (VCPU_2_VMXSTATE(pVCpu).enmShadowMode)
2426	{
2427	case PGMMODE_REAL: /* Real-mode. */
2428	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2429	case PGMMODE_32_BIT: /* 32-bit paging. */
2430	{
2431	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
2432	break;
2433	}
2434
2435	case PGMMODE_PAE: /* PAE paging. */
2436	case PGMMODE_PAE_NX: /* PAE paging with NX. */
2437	{
2438	u64GuestCr4 \|= X86_CR4_PAE;
2439	break;
2440	}
2441
2442	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2443	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2444	{
2445	#ifdef VBOX_WITH_64_BITS_GUESTS
2446	/* For our assumption in vmxHCShouldSwapEferMsr. */
2447	Assert(u64GuestCr4 & X86_CR4_PAE);
2448	break;
2449	#endif
2450	}
2451	default:
2452	AssertFailed();
2453	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2454	}
2455	}
2456
2457	/* Apply the hardware specified CR4 fixed bits (mainly CR4.VMXE). */
2458	u64GuestCr4 \|= fSetCr4;
2459	u64GuestCr4 &= fZapCr4;
2460
2461	/* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
2462	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR4, u64GuestCr4); AssertRC(rc);
2463	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4); AssertRC(rc);
2464
2465	#ifndef IN_NEM_DARWIN
2466	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
2467	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
2468	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
2469	{
2470	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
2471	hmR0VmxUpdateStartVmFunction(pVCpu);
2472	}
2473	#endif
2474
2475	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CR4);
2476
2477	Log4Func(("cr4=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
2478	}
2479	return rc;
2480	}
2481
2482
2483	#ifdef VBOX_STRICT
2484	/**
2485	* Strict function to validate segment registers.
2486	*
2487	* @param pVCpu The cross context virtual CPU structure.
2488	* @param pVmcsInfo The VMCS info. object.
2489	*
2490	* @remarks Will import guest CR0 on strict builds during validation of
2491	* segments.
2492	*/
2493	static void vmxHCValidateSegmentRegs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2494	{
2495	/*
2496	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
2497	*
2498	* The reason we check for attribute value 0 in this function and not just the unusable bit is
2499	* because vmxHCExportGuestSegReg() only updates the VMCS' copy of the value with the
2500	* unusable bit and doesn't change the guest-context value.
2501	*/
2502	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2503	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2504	vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
2505	if ( !VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
2506	&& ( !CPUMIsGuestInRealModeEx(pCtx)
2507	&& !CPUMIsGuestInV86ModeEx(pCtx)))
2508	{
2509	/* Protected mode checks */
2510	/* CS */
2511	Assert(pCtx->cs.Attr.n.u1Present);
2512	Assert(!(pCtx->cs.Attr.u & 0xf00));
2513	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
2514	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
2515	\|\| !(pCtx->cs.Attr.n.u1Granularity));
2516	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
2517	\|\| (pCtx->cs.Attr.n.u1Granularity));
2518	/* CS cannot be loaded with NULL in protected mode. */
2519	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
2520	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
2521	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
2522	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
2523	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
2524	else
2525	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
2526	/* SS */
2527	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
2528	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
2529	if ( !(pCtx->cr0 & X86_CR0_PE)
2530	\|\| pCtx->cs.Attr.n.u4Type == 3)
2531	{
2532	Assert(!pCtx->ss.Attr.n.u2Dpl);
2533	}
2534	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
2535	{
2536	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
2537	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
2538	Assert(pCtx->ss.Attr.n.u1Present);
2539	Assert(!(pCtx->ss.Attr.u & 0xf00));
2540	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
2541	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
2542	\|\| !(pCtx->ss.Attr.n.u1Granularity));
2543	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
2544	\|\| (pCtx->ss.Attr.n.u1Granularity));
2545	}
2546	/* DS, ES, FS, GS - only check for usable selectors, see vmxHCExportGuestSegReg(). */
2547	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
2548	{
2549	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
2550	Assert(pCtx->ds.Attr.n.u1Present);
2551	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
2552	Assert(!(pCtx->ds.Attr.u & 0xf00));
2553	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
2554	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
2555	\|\| !(pCtx->ds.Attr.n.u1Granularity));
2556	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
2557	\|\| (pCtx->ds.Attr.n.u1Granularity));
2558	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
2559	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
2560	}
2561	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
2562	{
2563	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
2564	Assert(pCtx->es.Attr.n.u1Present);
2565	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
2566	Assert(!(pCtx->es.Attr.u & 0xf00));
2567	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
2568	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
2569	\|\| !(pCtx->es.Attr.n.u1Granularity));
2570	Assert( !(pCtx->es.u32Limit & 0xfff00000)
2571	\|\| (pCtx->es.Attr.n.u1Granularity));
2572	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
2573	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
2574	}
2575	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
2576	{
2577	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
2578	Assert(pCtx->fs.Attr.n.u1Present);
2579	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
2580	Assert(!(pCtx->fs.Attr.u & 0xf00));
2581	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
2582	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
2583	\|\| !(pCtx->fs.Attr.n.u1Granularity));
2584	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
2585	\|\| (pCtx->fs.Attr.n.u1Granularity));
2586	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
2587	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
2588	}
2589	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
2590	{
2591	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
2592	Assert(pCtx->gs.Attr.n.u1Present);
2593	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
2594	Assert(!(pCtx->gs.Attr.u & 0xf00));
2595	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
2596	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
2597	\|\| !(pCtx->gs.Attr.n.u1Granularity));
2598	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
2599	\|\| (pCtx->gs.Attr.n.u1Granularity));
2600	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
2601	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
2602	}
2603	/* 64-bit capable CPUs. */
2604	Assert(!RT_HI_U32(pCtx->cs.u64Base));
2605	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
2606	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
2607	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
2608	}
2609	else if ( CPUMIsGuestInV86ModeEx(pCtx)
2610	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
2611	&& !VM_IS_VMX_UNRESTRICTED_GUEST(pVM)))
2612	{
2613	/* Real and v86 mode checks. */
2614	/* vmxHCExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
2615	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
2616	#ifndef IN_NEM_DARWIN
2617	if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
2618	{
2619	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
2620	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
2621	}
2622	else
2623	#endif
2624	{
2625	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
2626	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
2627	}
2628
2629	/* CS */
2630	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
2631	Assert(pCtx->cs.u32Limit == 0xffff);
2632	Assert(u32CSAttr == 0xf3);
2633	/* SS */
2634	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
2635	Assert(pCtx->ss.u32Limit == 0xffff);
2636	Assert(u32SSAttr == 0xf3);
2637	/* DS */
2638	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
2639	Assert(pCtx->ds.u32Limit == 0xffff);
2640	Assert(u32DSAttr == 0xf3);
2641	/* ES */
2642	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
2643	Assert(pCtx->es.u32Limit == 0xffff);
2644	Assert(u32ESAttr == 0xf3);
2645	/* FS */
2646	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
2647	Assert(pCtx->fs.u32Limit == 0xffff);
2648	Assert(u32FSAttr == 0xf3);
2649	/* GS */
2650	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
2651	Assert(pCtx->gs.u32Limit == 0xffff);
2652	Assert(u32GSAttr == 0xf3);
2653	/* 64-bit capable CPUs. */
2654	Assert(!RT_HI_U32(pCtx->cs.u64Base));
2655	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
2656	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
2657	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
2658	}
2659	}
2660	#endif /* VBOX_STRICT */
2661
2662
2663	/**
2664	* Exports a guest segment register into the guest-state area in the VMCS.
2665	*
2666	* @returns VBox status code.
2667	* @param pVCpu The cross context virtual CPU structure.
2668	* @param pVmcsInfo The VMCS info. object.
2669	* @param iSegReg The segment register number (X86_SREG_XXX).
2670	* @param pSelReg Pointer to the segment selector.
2671	*
2672	* @remarks No-long-jump zone!!!
2673	*/
2674	static int vmxHCExportGuestSegReg(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, uint32_t iSegReg, PCCPUMSELREG pSelReg)
2675	{
2676	Assert(iSegReg < X86_SREG_COUNT);
2677
2678	uint32_t u32Access = pSelReg->Attr.u;
2679	#ifndef IN_NEM_DARWIN
2680	if (!pVmcsInfo->pShared->RealMode.fRealOnV86Active)
2681	#endif
2682	{
2683	/*
2684	* The way to differentiate between whether this is really a null selector or was just
2685	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
2686	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
2687	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
2688	* NULL selectors loaded in protected-mode have their attribute as 0.
2689	*/
2690	if (u32Access)
2691	{ }
2692	else
2693	u32Access = X86DESCATTR_UNUSABLE;
2694	}
2695	#ifndef IN_NEM_DARWIN
2696	else
2697	{
2698	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
2699	u32Access = 0xf3;
2700	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2701	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2702	RT_NOREF_PV(pVCpu);
2703	}
2704	#else
2705	RT_NOREF(pVmcsInfo);
2706	#endif
2707
2708	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
2709	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
2710	("Access bit not set for usable segment. %.2s sel=%#x attr %#x\n", "ESCSSSDSFSGS" + iSegReg * 2, pSelReg, pSelReg->Attr.u));
2711
2712	/*
2713	* Commit it to the VMCS.
2714	*/
2715	Assert((uint32_t)VMX_VMCS16_GUEST_SEG_SEL(iSegReg) == g_aVmcsSegSel[iSegReg]);
2716	Assert((uint32_t)VMX_VMCS32_GUEST_SEG_LIMIT(iSegReg) == g_aVmcsSegLimit[iSegReg]);
2717	Assert((uint32_t)VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(iSegReg) == g_aVmcsSegAttr[iSegReg]);
2718	Assert((uint32_t)VMX_VMCS_GUEST_SEG_BASE(iSegReg) == g_aVmcsSegBase[iSegReg]);
2719	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS16_GUEST_SEG_SEL(iSegReg), pSelReg->Sel); AssertRC(rc);
2720	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_SEG_LIMIT(iSegReg), pSelReg->u32Limit); AssertRC(rc);
2721	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_SEG_BASE(iSegReg), pSelReg->u64Base); AssertRC(rc);
2722	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(iSegReg), u32Access); AssertRC(rc);
2723	return VINF_SUCCESS;
2724	}
2725
2726
2727	/**
2728	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
2729	* area in the VMCS.
2730	*
2731	* @returns VBox status code.
2732	* @param pVCpu The cross context virtual CPU structure.
2733	* @param pVmxTransient The VMX-transient structure.
2734	*
2735	* @remarks Will import guest CR0 on strict builds during validation of
2736	* segments.
2737	* @remarks No-long-jump zone!!!
2738	*/
2739	static int vmxHCExportGuestSegRegsXdtr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
2740	{
2741	int rc = VERR_INTERNAL_ERROR_5;
2742	#ifndef IN_NEM_DARWIN
2743	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2744	#endif
2745	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2746	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2747	#ifndef IN_NEM_DARWIN
2748	PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
2749	#endif
2750
2751	/*
2752	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
2753	*/
2754	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
2755	{
2756	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CS)
2757	{
2758	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
2759	#ifndef IN_NEM_DARWIN
2760	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
2761	pVmcsInfoShared->RealMode.AttrCS.u = pCtx->cs.Attr.u;
2762	#endif
2763	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
2764	AssertRC(rc);
2765	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CS);
2766	}
2767
2768	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_SS)
2769	{
2770	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
2771	#ifndef IN_NEM_DARWIN
2772	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
2773	pVmcsInfoShared->RealMode.AttrSS.u = pCtx->ss.Attr.u;
2774	#endif
2775	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
2776	AssertRC(rc);
2777	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_SS);
2778	}
2779
2780	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_DS)
2781	{
2782	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
2783	#ifndef IN_NEM_DARWIN
2784	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
2785	pVmcsInfoShared->RealMode.AttrDS.u = pCtx->ds.Attr.u;
2786	#endif
2787	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
2788	AssertRC(rc);
2789	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_DS);
2790	}
2791
2792	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_ES)
2793	{
2794	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
2795	#ifndef IN_NEM_DARWIN
2796	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
2797	pVmcsInfoShared->RealMode.AttrES.u = pCtx->es.Attr.u;
2798	#endif
2799	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
2800	AssertRC(rc);
2801	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_ES);
2802	}
2803
2804	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_FS)
2805	{
2806	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
2807	#ifndef IN_NEM_DARWIN
2808	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
2809	pVmcsInfoShared->RealMode.AttrFS.u = pCtx->fs.Attr.u;
2810	#endif
2811	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
2812	AssertRC(rc);
2813	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_FS);
2814	}
2815
2816	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_GS)
2817	{
2818	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
2819	#ifndef IN_NEM_DARWIN
2820	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
2821	pVmcsInfoShared->RealMode.AttrGS.u = pCtx->gs.Attr.u;
2822	#endif
2823	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
2824	AssertRC(rc);
2825	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_GS);
2826	}
2827
2828	#ifdef VBOX_STRICT
2829	vmxHCValidateSegmentRegs(pVCpu, pVmcsInfo);
2830	#endif
2831	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
2832	pCtx->cs.Attr.u));
2833	}
2834
2835	/*
2836	* Guest TR.
2837	*/
2838	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_TR)
2839	{
2840	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
2841
2842	/*
2843	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
2844	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
2845	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
2846	*/
2847	uint16_t u16Sel;
2848	uint32_t u32Limit;
2849	uint64_t u64Base;
2850	uint32_t u32AccessRights;
2851	#ifndef IN_NEM_DARWIN
2852	if (!pVmcsInfoShared->RealMode.fRealOnV86Active)
2853	#endif
2854	{
2855	u16Sel = pCtx->tr.Sel;
2856	u32Limit = pCtx->tr.u32Limit;
2857	u64Base = pCtx->tr.u64Base;
2858	u32AccessRights = pCtx->tr.Attr.u;
2859	}
2860	#ifndef IN_NEM_DARWIN
2861	else
2862	{
2863	Assert(!pVmxTransient->fIsNestedGuest);
2864	Assert(pVM->hm.s.vmx.pRealModeTSS);
2865	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
2866
2867	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
2868	RTGCPHYS GCPhys;
2869	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
2870	AssertRCReturn(rc, rc);
2871
2872	X86DESCATTR DescAttr;
2873	DescAttr.u = 0;
2874	DescAttr.n.u1Present = 1;
2875	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
2876
2877	u16Sel = 0;
2878	u32Limit = HM_VTX_TSS_SIZE;
2879	u64Base = GCPhys;
2880	u32AccessRights = DescAttr.u;
2881	}
2882	#endif
2883
2884	/* Validate. */
2885	Assert(!(u16Sel & RT_BIT(2)));
2886	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
2887	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
2888	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
2889	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
2890	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
2891	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
2892	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
2893	Assert( (u32Limit & 0xfff) == 0xfff
2894	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
2895	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
2896	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
2897
2898	rc = VMX_VMCS_WRITE_16(pVCpu, VMX_VMCS16_GUEST_TR_SEL, u16Sel); AssertRC(rc);
2899	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRC(rc);
2900	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRC(rc);
2901	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRC(rc);
2902
2903	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_TR);
2904	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
2905	}
2906
2907	/*
2908	* Guest GDTR.
2909	*/
2910	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_GDTR)
2911	{
2912	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
2913
2914	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt); AssertRC(rc);
2915	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt); AssertRC(rc);
2916
2917	/* Validate. */
2918	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
2919
2920	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
2921	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
2922	}
2923
2924	/*
2925	* Guest LDTR.
2926	*/
2927	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_LDTR)
2928	{
2929	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
2930
2931	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
2932	uint32_t u32Access;
2933	if ( !pVmxTransient->fIsNestedGuest
2934	&& !pCtx->ldtr.Attr.u)
2935	u32Access = X86DESCATTR_UNUSABLE;
2936	else
2937	u32Access = pCtx->ldtr.Attr.u;
2938
2939	rc = VMX_VMCS_WRITE_16(pVCpu, VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel); AssertRC(rc);
2940	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit); AssertRC(rc);
2941	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRC(rc);
2942	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base); AssertRC(rc);
2943
2944	/* Validate. */
2945	if (!(u32Access & X86DESCATTR_UNUSABLE))
2946	{
2947	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
2948	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
2949	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
2950	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
2951	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
2952	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
2953	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
2954	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
2955	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
2956	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
2957	}
2958
2959	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
2960	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
2961	}
2962
2963	/*
2964	* Guest IDTR.
2965	*/
2966	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_IDTR)
2967	{
2968	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
2969
2970	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt); AssertRC(rc);
2971	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt); AssertRC(rc);
2972
2973	/* Validate. */
2974	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
2975
2976	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
2977	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
2978	}
2979
2980	return VINF_SUCCESS;
2981	}
2982
2983
2984	/**
2985	* Gets the IEM exception flags for the specified vector and IDT vectoring /
2986	* VM-exit interruption info type.
2987	*
2988	* @returns The IEM exception flags.
2989	* @param uVector The event vector.
2990	* @param uVmxEventType The VMX event type.
2991	*
2992	* @remarks This function currently only constructs flags required for
2993	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
2994	* and CR2 aspects of an exception are not included).
2995	*/
2996	static uint32_t vmxHCGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
2997	{
2998	uint32_t fIemXcptFlags;
2999	switch (uVmxEventType)
3000	{
3001	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
3002	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
3003	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
3004	break;
3005
3006	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
3007	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
3008	break;
3009
3010	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
3011	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
3012	break;
3013
3014	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
3015	{
3016	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
3017	if (uVector == X86_XCPT_BP)
3018	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
3019	else if (uVector == X86_XCPT_OF)
3020	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
3021	else
3022	{
3023	fIemXcptFlags = 0;
3024	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
3025	}
3026	break;
3027	}
3028
3029	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
3030	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
3031	break;
3032
3033	default:
3034	fIemXcptFlags = 0;
3035	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
3036	break;
3037	}
3038	return fIemXcptFlags;
3039	}
3040
3041
3042	/**
3043	* Sets an event as a pending event to be injected into the guest.
3044	*
3045	* @param pVCpu The cross context virtual CPU structure.
3046	* @param u32IntInfo The VM-entry interruption-information field.
3047	* @param cbInstr The VM-entry instruction length in bytes (for
3048	* software interrupts, exceptions and privileged
3049	* software exceptions).
3050	* @param u32ErrCode The VM-entry exception error code.
3051	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
3052	* page-fault.
3053	*/
3054	DECLINLINE(void) vmxHCSetPendingEvent(PVMCPUCC pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
3055	RTGCUINTPTR GCPtrFaultAddress)
3056	{
3057	Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending);
3058	VCPU_2_VMXSTATE(pVCpu).Event.fPending = true;
3059	VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo = u32IntInfo;
3060	VCPU_2_VMXSTATE(pVCpu).Event.u32ErrCode = u32ErrCode;
3061	VCPU_2_VMXSTATE(pVCpu).Event.cbInstr = cbInstr;
3062	VCPU_2_VMXSTATE(pVCpu).Event.GCPtrFaultAddress = GCPtrFaultAddress;
3063	}
3064
3065
3066	/**
3067	* Sets an external interrupt as pending-for-injection into the VM.
3068	*
3069	* @param pVCpu The cross context virtual CPU structure.
3070	* @param u8Interrupt The external interrupt vector.
3071	*/
3072	DECLINLINE(void) vmxHCSetPendingExtInt(PVMCPUCC pVCpu, uint8_t u8Interrupt)
3073	{
3074	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
3075	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
3076	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
3077	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3078	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
3079	}
3080
3081
3082	/**
3083	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
3084	*
3085	* @param pVCpu The cross context virtual CPU structure.
3086	*/
3087	DECLINLINE(void) vmxHCSetPendingXcptNmi(PVMCPUCC pVCpu)
3088	{
3089	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
3090	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
3091	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
3092	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3093	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
3094	}
3095
3096
3097	/**
3098	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
3099	*
3100	* @param pVCpu The cross context virtual CPU structure.
3101	*/
3102	DECLINLINE(void) vmxHCSetPendingXcptDF(PVMCPUCC pVCpu)
3103	{
3104	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
3105	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
3106	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
3107	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3108	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
3109	}
3110
3111
3112	/**
3113	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
3114	*
3115	* @param pVCpu The cross context virtual CPU structure.
3116	*/
3117	DECLINLINE(void) vmxHCSetPendingXcptUD(PVMCPUCC pVCpu)
3118	{
3119	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
3120	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
3121	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
3122	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3123	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
3124	}
3125
3126
3127	/**
3128	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
3129	*
3130	* @param pVCpu The cross context virtual CPU structure.
3131	*/
3132	DECLINLINE(void) vmxHCSetPendingXcptDB(PVMCPUCC pVCpu)
3133	{
3134	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
3135	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
3136	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
3137	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3138	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
3139	}
3140
3141
3142	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3143	/**
3144	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
3145	*
3146	* @param pVCpu The cross context virtual CPU structure.
3147	* @param u32ErrCode The error code for the general-protection exception.
3148	*/
3149	DECLINLINE(void) vmxHCSetPendingXcptGP(PVMCPUCC pVCpu, uint32_t u32ErrCode)
3150	{
3151	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
3152	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
3153	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
3154	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3155	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
3156	}
3157
3158
3159	/**
3160	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
3161	*
3162	* @param pVCpu The cross context virtual CPU structure.
3163	* @param u32ErrCode The error code for the stack exception.
3164	*/
3165	DECLINLINE(void) vmxHCSetPendingXcptSS(PVMCPUCC pVCpu, uint32_t u32ErrCode)
3166	{
3167	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
3168	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
3169	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
3170	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3171	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
3172	}
3173	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3174
3175
3176	/**
3177	* Fixes up attributes for the specified segment register.
3178	*
3179	* @param pVCpu The cross context virtual CPU structure.
3180	* @param pSelReg The segment register that needs fixing.
3181	* @param pszRegName The register name (for logging and assertions).
3182	*/
3183	static void vmxHCFixUnusableSegRegAttr(PVMCPUCC pVCpu, PCPUMSELREG pSelReg, const char *pszRegName)
3184	{
3185	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
3186
3187	/*
3188	* If VT-x marks the segment as unusable, most other bits remain undefined:
3189	* - For CS the L, D and G bits have meaning.
3190	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
3191	* - For the remaining data segments no bits are defined.
3192	*
3193	* The present bit and the unusable bit has been observed to be set at the
3194	* same time (the selector was supposed to be invalid as we started executing
3195	* a V8086 interrupt in ring-0).
3196	*
3197	* What should be important for the rest of the VBox code, is that the P bit is
3198	* cleared. Some of the other VBox code recognizes the unusable bit, but
3199	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
3200	* safe side here, we'll strip off P and other bits we don't care about. If
3201	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
3202	*
3203	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
3204	*/
3205	#ifdef VBOX_STRICT
3206	uint32_t const uAttr = pSelReg->Attr.u;
3207	#endif
3208
3209	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
3210	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
3211	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
3212
3213	#ifdef VBOX_STRICT
3214	# ifndef IN_NEM_DARWIN
3215	VMMRZCallRing3Disable(pVCpu);
3216	# endif
3217	Log4Func(("Unusable %s: sel=%#x attr=%#x -> %#x\n", pszRegName, pSelReg->Sel, uAttr, pSelReg->Attr.u));
3218	# ifdef DEBUG_bird
3219	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
3220	("%s: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
3221	pszRegName, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
3222	# endif
3223	# ifndef IN_NEM_DARWIN
3224	VMMRZCallRing3Enable(pVCpu);
3225	# endif
3226	NOREF(uAttr);
3227	#endif
3228	RT_NOREF2(pVCpu, pszRegName);
3229	}
3230
3231
3232	/**
3233	* Imports a guest segment register from the current VMCS into the guest-CPU
3234	* context.
3235	*
3236	* @param pVCpu The cross context virtual CPU structure.
3237	* @tparam a_iSegReg The segment register number (X86_SREG_XXX).
3238	*
3239	* @remarks Called with interrupts and/or preemption disabled.
3240	*/
3241	template<uint32_t const a_iSegReg>
3242	DECLINLINE(void) vmxHCImportGuestSegReg(PVMCPUCC pVCpu)
3243	{
3244	AssertCompile(a_iSegReg < X86_SREG_COUNT);
3245	Assert((uint32_t)VMX_VMCS16_GUEST_SEG_SEL(a_iSegReg) == g_aVmcsSegSel[a_iSegReg]);
3246	Assert((uint32_t)VMX_VMCS32_GUEST_SEG_LIMIT(a_iSegReg) == g_aVmcsSegLimit[a_iSegReg]);
3247	Assert((uint32_t)VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(a_iSegReg) == g_aVmcsSegAttr[a_iSegReg]);
3248	Assert((uint32_t)VMX_VMCS_GUEST_SEG_BASE(a_iSegReg) == g_aVmcsSegBase[a_iSegReg]);
3249
3250	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[a_iSegReg];
3251
3252	uint16_t u16Sel;
3253	int rc = VMX_VMCS_READ_16(pVCpu, VMX_VMCS16_GUEST_SEG_SEL(a_iSegReg), &u16Sel); AssertRC(rc);
3254	pSelReg->Sel = u16Sel;
3255	pSelReg->ValidSel = u16Sel;
3256
3257	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SEG_LIMIT(a_iSegReg), &pSelReg->u32Limit); AssertRC(rc);
3258	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SEG_BASE(a_iSegReg), &pSelReg->u64Base); AssertRC(rc);
3259
3260	uint32_t u32Attr;
3261	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(a_iSegReg), &u32Attr); AssertRC(rc);
3262	pSelReg->Attr.u = u32Attr;
3263	if (u32Attr & X86DESCATTR_UNUSABLE)
3264	vmxHCFixUnusableSegRegAttr(pVCpu, pSelReg, "ES\0CS\0SS\0DS\0FS\0GS" + a_iSegReg * 3);
3265
3266	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
3267	}
3268
3269
3270	/**
3271	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
3272	*
3273	* @param pVCpu The cross context virtual CPU structure.
3274	*
3275	* @remarks Called with interrupts and/or preemption disabled.
3276	*/
3277	static void vmxHCImportGuestLdtr(PVMCPUCC pVCpu)
3278	{
3279	uint16_t u16Sel;
3280	uint64_t u64Base;
3281	uint32_t u32Limit, u32Attr;
3282	int rc = VMX_VMCS_READ_16(pVCpu, VMX_VMCS16_GUEST_LDTR_SEL, &u16Sel); AssertRC(rc);
3283	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit); AssertRC(rc);
3284	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc);
3285	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_LDTR_BASE, &u64Base); AssertRC(rc);
3286
3287	pVCpu->cpum.GstCtx.ldtr.Sel = u16Sel;
3288	pVCpu->cpum.GstCtx.ldtr.ValidSel = u16Sel;
3289	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
3290	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
3291	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
3292	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
3293	if (u32Attr & X86DESCATTR_UNUSABLE)
3294	vmxHCFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, "LDTR");
3295	}
3296
3297
3298	/**
3299	* Imports the guest TR from the current VMCS into the guest-CPU context.
3300	*
3301	* @param pVCpu The cross context virtual CPU structure.
3302	*
3303	* @remarks Called with interrupts and/or preemption disabled.
3304	*/
3305	DECLINLINE(void) vmxHCImportGuestTr(PVMCPUCC pVCpu)
3306	{
3307	uint16_t u16Sel;
3308	uint64_t u64Base;
3309	uint32_t u32Limit, u32Attr;
3310	int rc = VMX_VMCS_READ_16(pVCpu, VMX_VMCS16_GUEST_TR_SEL, &u16Sel); AssertRC(rc);
3311	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit); AssertRC(rc);
3312	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc);
3313	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_TR_BASE, &u64Base); AssertRC(rc);
3314
3315	pVCpu->cpum.GstCtx.tr.Sel = u16Sel;
3316	pVCpu->cpum.GstCtx.tr.ValidSel = u16Sel;
3317	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
3318	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
3319	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
3320	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
3321	/* TR is the only selector that can never be unusable. */
3322	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
3323	}
3324
3325
3326	/**
3327	* Imports the guest RIP from the VMCS back into the guest-CPU context.
3328	*
3329	* @param pVCpu The cross context virtual CPU structure.
3330	*
3331	* @remarks Called with interrupts and/or preemption disabled, should not assert!
3332	* @remarks Do -not- call this function directly, use vmxHCImportGuestState()
3333	* instead!!!
3334	*/
3335	DECLINLINE(void) vmxHCImportGuestRip(PVMCPUCC pVCpu)
3336	{
3337	uint64_t u64Val;
3338	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3339	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
3340	{
3341	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RIP, &u64Val);
3342	AssertRC(rc);
3343
3344	pCtx->rip = u64Val;
3345	EMHistoryUpdatePC(pVCpu, pCtx->rip, false);
3346	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
3347	}
3348	}
3349
3350
3351	/**
3352	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
3353	*
3354	* @param pVCpu The cross context virtual CPU structure.
3355	* @param pVmcsInfo The VMCS info. object.
3356	*
3357	* @remarks Called with interrupts and/or preemption disabled, should not assert!
3358	* @remarks Do -not- call this function directly, use vmxHCImportGuestState()
3359	* instead!!!
3360	*/
3361	DECLINLINE(void) vmxHCImportGuestRFlags(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
3362	{
3363	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3364	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
3365	{
3366	uint64_t u64Val;
3367	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RFLAGS, &u64Val);
3368	AssertRC(rc);
3369
3370	pCtx->rflags.u64 = u64Val;
3371	#ifndef IN_NEM_DARWIN
3372	PCVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
3373	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
3374	{
3375	pCtx->eflags.Bits.u1VM = 0;
3376	pCtx->eflags.Bits.u2IOPL = pVmcsInfoShared->RealMode.Eflags.Bits.u2IOPL;
3377	}
3378	#else
3379	RT_NOREF(pVmcsInfo);
3380	#endif
3381	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
3382	}
3383	}
3384
3385
3386	/**
3387	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
3388	* context.
3389	*
3390	* @param pVCpu The cross context virtual CPU structure.
3391	* @param pVmcsInfo The VMCS info. object.
3392	*
3393	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
3394	* do not log!
3395	* @remarks Do -not- call this function directly, use vmxHCImportGuestState()
3396	* instead!!!
3397	*/
3398	DECLINLINE(void) vmxHCImportGuestIntrState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
3399	{
3400	uint32_t u32Val;
3401	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &u32Val); AssertRC(rc);
3402	if (!u32Val)
3403	{
3404	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3405	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3406	CPUMSetGuestNmiBlocking(pVCpu, false);
3407	}
3408	else
3409	{
3410	/*
3411	* We must import RIP here to set our EM interrupt-inhibited state.
3412	* We also import RFLAGS as our code that evaluates pending interrupts
3413	* before VM-entry requires it.
3414	*/
3415	vmxHCImportGuestRip(pVCpu);
3416	vmxHCImportGuestRFlags(pVCpu, pVmcsInfo);
3417
3418	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
3419	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
3420	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3421	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3422
3423	bool const fNmiBlocking = RT_BOOL(u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
3424	CPUMSetGuestNmiBlocking(pVCpu, fNmiBlocking);
3425	}
3426	}
3427
3428
3429	/**
3430	* Worker for VMXR0ImportStateOnDemand.
3431	*
3432	* @returns VBox status code.
3433	* @param pVCpu The cross context virtual CPU structure.
3434	* @param pVmcsInfo The VMCS info. object.
3435	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
3436	*/
3437	static int vmxHCImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
3438	{
3439	int rc = VINF_SUCCESS;
3440	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3441	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3442	uint32_t u32Val;
3443
3444	/*
3445	* Note! This is hack to workaround a mysterious BSOD observed with release builds
3446	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
3447	* neither are other host platforms.
3448	*
3449	* Committing this temporarily as it prevents BSOD.
3450	*
3451	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
3452	*/
3453	# ifdef RT_OS_WINDOWS
3454	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
3455	return VERR_HM_IPE_1;
3456	# endif
3457
3458	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x);
3459
3460	#ifndef IN_NEM_DARWIN
3461	/*
3462	* We disable interrupts to make the updating of the state and in particular
3463	* the fExtrn modification atomic wrt to preemption hooks.
3464	*/
3465	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
3466	#endif
3467
3468	fWhat &= pCtx->fExtrn;
3469	if (fWhat)
3470	{
3471	do
3472	{
3473	if (fWhat & CPUMCTX_EXTRN_RIP)
3474	vmxHCImportGuestRip(pVCpu);
3475
3476	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
3477	vmxHCImportGuestRFlags(pVCpu, pVmcsInfo);
3478
3479	if (fWhat & (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI))
3480	vmxHCImportGuestIntrState(pVCpu, pVmcsInfo);
3481
3482	if (fWhat & CPUMCTX_EXTRN_RSP)
3483	{
3484	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RSP, &pCtx->rsp);
3485	AssertRC(rc);
3486	}
3487
3488	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
3489	{
3490	PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
3491	#ifndef IN_NEM_DARWIN
3492	bool const fRealOnV86Active = pVmcsInfoShared->RealMode.fRealOnV86Active;
3493	#else
3494	bool const fRealOnV86Active = false; /* HV supports only unrestricted guest execution. */
3495	#endif
3496	if (fWhat & CPUMCTX_EXTRN_CS)
3497	{
3498	vmxHCImportGuestSegReg<X86_SREG_CS>(pVCpu);
3499	vmxHCImportGuestRip(pVCpu); /** @todo WTF? */
3500	if (fRealOnV86Active)
3501	pCtx->cs.Attr.u = pVmcsInfoShared->RealMode.AttrCS.u;
3502	EMHistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
3503	}
3504	if (fWhat & CPUMCTX_EXTRN_SS)
3505	{
3506	vmxHCImportGuestSegReg<X86_SREG_SS>(pVCpu);
3507	if (fRealOnV86Active)
3508	pCtx->ss.Attr.u = pVmcsInfoShared->RealMode.AttrSS.u;
3509	}
3510	if (fWhat & CPUMCTX_EXTRN_DS)
3511	{
3512	vmxHCImportGuestSegReg<X86_SREG_DS>(pVCpu);
3513	if (fRealOnV86Active)
3514	pCtx->ds.Attr.u = pVmcsInfoShared->RealMode.AttrDS.u;
3515	}
3516	if (fWhat & CPUMCTX_EXTRN_ES)
3517	{
3518	vmxHCImportGuestSegReg<X86_SREG_ES>(pVCpu);
3519	if (fRealOnV86Active)
3520	pCtx->es.Attr.u = pVmcsInfoShared->RealMode.AttrES.u;
3521	}
3522	if (fWhat & CPUMCTX_EXTRN_FS)
3523	{
3524	vmxHCImportGuestSegReg<X86_SREG_FS>(pVCpu);
3525	if (fRealOnV86Active)
3526	pCtx->fs.Attr.u = pVmcsInfoShared->RealMode.AttrFS.u;
3527	}
3528	if (fWhat & CPUMCTX_EXTRN_GS)
3529	{
3530	vmxHCImportGuestSegReg<X86_SREG_GS>(pVCpu);
3531	if (fRealOnV86Active)
3532	pCtx->gs.Attr.u = pVmcsInfoShared->RealMode.AttrGS.u;
3533	}
3534	}
3535
3536	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
3537	{
3538	if (fWhat & CPUMCTX_EXTRN_LDTR)
3539	vmxHCImportGuestLdtr(pVCpu);
3540
3541	if (fWhat & CPUMCTX_EXTRN_GDTR)
3542	{
3543	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE, &pCtx->gdtr.pGdt); AssertRC(rc);
3544	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRC(rc);
3545	pCtx->gdtr.cbGdt = u32Val;
3546	}
3547
3548	/* Guest IDTR. */
3549	if (fWhat & CPUMCTX_EXTRN_IDTR)
3550	{
3551	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE, &pCtx->idtr.pIdt); AssertRC(rc);
3552	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRC(rc);
3553	pCtx->idtr.cbIdt = u32Val;
3554	}
3555
3556	/* Guest TR. */
3557	if (fWhat & CPUMCTX_EXTRN_TR)
3558	{
3559	#ifndef IN_NEM_DARWIN
3560	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
3561	don't need to import that one. */
3562	if (!pVmcsInfo->pShared->RealMode.fRealOnV86Active)
3563	#endif
3564	vmxHCImportGuestTr(pVCpu);
3565	}
3566	}
3567
3568	if (fWhat & CPUMCTX_EXTRN_DR7)
3569	{
3570	#ifndef IN_NEM_DARWIN
3571	if (!pVCpu->hmr0.s.fUsingHyperDR7)
3572	#endif
3573	{
3574	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_DR7, &pCtx->dr[7]);
3575	AssertRC(rc);
3576	}
3577	}
3578
3579	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
3580	{
3581	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip); AssertRC(rc);
3582	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp); AssertRC(rc);
3583	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRC(rc);
3584	pCtx->SysEnter.cs = u32Val;
3585	}
3586
3587	#ifndef IN_NEM_DARWIN
3588	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
3589	{
3590	if ( pVM->hmr0.s.fAllow64BitGuests
3591	&& (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
3592	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
3593	}
3594
3595	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
3596	{
3597	if ( pVM->hmr0.s.fAllow64BitGuests
3598	&& (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
3599	{
3600	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3601	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
3602	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
3603	}
3604	}
3605
3606	if (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
3607	{
3608	PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
3609	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
3610	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
3611	Assert(pMsrs);
3612	Assert(cMsrs <= VMX_MISC_MAX_MSRS(g_HmMsrs.u.vmx.u64Misc));
3613	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
3614	for (uint32_t i = 0; i < cMsrs; i++)
3615	{
3616	uint32_t const idMsr = pMsrs[i].u32Msr;
3617	switch (idMsr)
3618	{
3619	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
3620	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
3621	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
3622	default:
3623	{
3624	uint32_t idxLbrMsr;
3625	if (VM_IS_VMX_LBR(pVM))
3626	{
3627	if (hmR0VmxIsLbrBranchFromMsr(pVM, idMsr, &idxLbrMsr))
3628	{
3629	Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr));
3630	pVmcsInfoShared->au64LbrFromIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
3631	break;
3632	}
3633	if (hmR0VmxIsLbrBranchToMsr(pVM, idMsr, &idxLbrMsr))
3634	{
3635	Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr));
3636	pVmcsInfoShared->au64LbrToIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
3637	break;
3638	}
3639	if (idMsr == pVM->hmr0.s.vmx.idLbrTosMsr)
3640	{
3641	pVmcsInfoShared->u64LbrTosMsr = pMsrs[i].u64Value;
3642	break;
3643	}
3644	/* Fallthru (no break) */
3645	}
3646	pCtx->fExtrn = 0;
3647	VCPU_2_VMXSTATE(pVCpu).u32HMError = pMsrs->u32Msr;
3648	ASMSetFlags(fEFlags);
3649	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
3650	return VERR_HM_UNEXPECTED_LD_ST_MSR;
3651	}
3652	}
3653	}
3654	}
3655	#endif
3656
3657	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
3658	{
3659	if (fWhat & CPUMCTX_EXTRN_CR0)
3660	{
3661	uint64_t u64Cr0;
3662	uint64_t u64Shadow;
3663	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0, &u64Cr0); AssertRC(rc);
3664	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow); AssertRC(rc);
3665	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
3666	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
3667	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
3668	#else
3669	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
3670	{
3671	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
3672	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
3673	}
3674	else
3675	{
3676	/*
3677	* We've merged the guest and nested-guest's CR0 guest/host mask while executing
3678	* the nested-guest using hardware-assisted VMX. Accordingly we need to
3679	* re-construct CR0. See @bugref{9180#c95} for details.
3680	*/
3681	PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo;
3682	PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
3683	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
3684	\| (pVmcsNstGst->u64GuestCr0.u & pVmcsNstGst->u64Cr0Mask.u)
3685	\| (u64Shadow & (pVmcsInfoGst->u64Cr0Mask & ~pVmcsNstGst->u64Cr0Mask.u));
3686	}
3687	#endif
3688	#ifndef IN_NEM_DARWIN
3689	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
3690	#endif
3691	CPUMSetGuestCR0(pVCpu, u64Cr0);
3692	#ifndef IN_NEM_DARWIN
3693	VMMRZCallRing3Enable(pVCpu);
3694	#endif
3695	}
3696
3697	if (fWhat & CPUMCTX_EXTRN_CR4)
3698	{
3699	uint64_t u64Cr4;
3700	uint64_t u64Shadow;
3701	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR4, &u64Cr4); AssertRC(rc);
3702	rc \|= VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow); AssertRC(rc);
3703	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
3704	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
3705	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
3706	#else
3707	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
3708	{
3709	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
3710	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
3711	}
3712	else
3713	{
3714	/*
3715	* We've merged the guest and nested-guest's CR4 guest/host mask while executing
3716	* the nested-guest using hardware-assisted VMX. Accordingly we need to
3717	* re-construct CR4. See @bugref{9180#c95} for details.
3718	*/
3719	PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo;
3720	PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
3721	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
3722	\| (pVmcsNstGst->u64GuestCr4.u & pVmcsNstGst->u64Cr4Mask.u)
3723	\| (u64Shadow & (pVmcsInfoGst->u64Cr4Mask & ~pVmcsNstGst->u64Cr4Mask.u));
3724	}
3725	#endif
3726	pCtx->cr4 = u64Cr4;
3727	}
3728
3729	if (fWhat & CPUMCTX_EXTRN_CR3)
3730	{
3731	/* CR0.PG bit changes are always intercepted, so it's up to date. */
3732	if ( VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
3733	\|\| ( VM_IS_VMX_NESTED_PAGING(pVM)
3734	&& CPUMIsGuestPagingEnabledEx(pCtx)))
3735	{
3736	uint64_t u64Cr3;
3737	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR3, &u64Cr3); AssertRC(rc);
3738	if (pCtx->cr3 != u64Cr3)
3739	{
3740	pCtx->cr3 = u64Cr3;
3741	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
3742	}
3743
3744	/*
3745	* If the guest is in PAE mode, sync back the PDPE's into the guest state.
3746	* CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date.
3747	*/
3748	if (CPUMIsGuestInPAEModeEx(pCtx))
3749	{
3750	X86PDPE aPaePdpes[4];
3751	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, &aPaePdpes[0].u); AssertRC(rc);
3752	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, &aPaePdpes[1].u); AssertRC(rc);
3753	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, &aPaePdpes[2].u); AssertRC(rc);
3754	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, &aPaePdpes[3].u); AssertRC(rc);
3755	if (memcmp(&aPaePdpes[0], &pCtx->aPaePdpes[0], sizeof(aPaePdpes)))
3756	{
3757	memcpy(&pCtx->aPaePdpes[0], &aPaePdpes[0], sizeof(aPaePdpes));
3758	/* PGM now updates PAE PDPTEs while updating CR3. */
3759	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
3760	}
3761	}
3762	}
3763	}
3764	}
3765
3766	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3767	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
3768	{
3769	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
3770	&& !CPUMIsGuestInVmxNonRootMode(pCtx))
3771	{
3772	Assert(CPUMIsGuestInVmxRootMode(pCtx));
3773	rc = vmxHCCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
3774	if (RT_SUCCESS(rc))
3775	{ /* likely */ }
3776	else
3777	break;
3778	}
3779	}
3780	#endif
3781	} while (0);
3782
3783	if (RT_SUCCESS(rc))
3784	{
3785	/* Update fExtrn. */
3786	pCtx->fExtrn &= ~fWhat;
3787
3788	/* If everything has been imported, clear the HM keeper bit. */
3789	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
3790	{
3791	#ifndef IN_NEM_DARWIN
3792	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
3793	#else
3794	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_NEM;
3795	#endif
3796	Assert(!pCtx->fExtrn);
3797	}
3798	}
3799	}
3800	#ifndef IN_NEM_DARWIN
3801	else
3802	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
3803
3804	/*
3805	* Restore interrupts.
3806	*/
3807	ASMSetFlags(fEFlags);
3808	#endif
3809
3810	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x);
3811
3812	if (RT_SUCCESS(rc))
3813	{ /* likely */ }
3814	else
3815	return rc;
3816
3817	/*
3818	* Honor any pending CR3 updates.
3819	*
3820	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> VMXR0CallRing3Callback()
3821	* -> VMMRZCallRing3Disable() -> vmxHCImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
3822	* -> continue with VM-exit handling -> vmxHCImportGuestState() and here we are.
3823	*
3824	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
3825	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
3826	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
3827	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
3828	*
3829	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
3830	*
3831	* The force-flag is checked first as it's cheaper for potential superfluous calls to this function.
3832	*/
3833	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3)
3834	#ifndef IN_NEM_DARWIN
3835	&& VMMRZCallRing3IsEnabled(pVCpu)
3836	#endif
3837	)
3838	{
3839	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
3840	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
3841	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
3842	}
3843
3844	return VINF_SUCCESS;
3845	}
3846
3847
3848	/**
3849	* Check per-VM and per-VCPU force flag actions that require us to go back to
3850	* ring-3 for one reason or another.
3851	*
3852	* @returns Strict VBox status code (i.e. informational status codes too)
3853	* @retval VINF_SUCCESS if we don't have any actions that require going back to
3854	* ring-3.
3855	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
3856	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
3857	* interrupts)
3858	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
3859	* all EMTs to be in ring-3.
3860	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
3861	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
3862	* to the EM loop.
3863	*
3864	* @param pVCpu The cross context virtual CPU structure.
3865	* @param fIsNestedGuest Flag whether this is for a for a pending nested guest event.
3866	* @param fStepping Whether we are single-stepping the guest using the
3867	* hypervisor debugger.
3868	*
3869	* @remarks This might cause nested-guest VM-exits, caller must check if the guest
3870	* is no longer in VMX non-root mode.
3871	*/
3872	static VBOXSTRICTRC vmxHCCheckForceFlags(PVMCPUCC pVCpu, bool fIsNestedGuest, bool fStepping)
3873	{
3874	#ifndef IN_NEM_DARWIN
3875	Assert(VMMRZCallRing3IsEnabled(pVCpu));
3876	#endif
3877
3878	/*
3879	* Update pending interrupts into the APIC's IRR.
3880	*/
3881	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
3882	APICUpdatePendingInterrupts(pVCpu);
3883
3884	/*
3885	* Anything pending? Should be more likely than not if we're doing a good job.
3886	*/
3887	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3888	if ( !fStepping
3889	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
3890	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
3891	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
3892	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
3893	return VINF_SUCCESS;
3894
3895	/* Pending PGM C3 sync. */
3896	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
3897	{
3898	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3899	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
3900	VBOXSTRICTRC rcStrict = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
3901	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
3902	if (rcStrict != VINF_SUCCESS)
3903	{
3904	AssertRC(VBOXSTRICTRC_VAL(rcStrict));
3905	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
3906	return rcStrict;
3907	}
3908	}
3909
3910	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
3911	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
3912	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
3913	{
3914	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchHmToR3FF);
3915	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
3916	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d (fVM=%#RX64 fCpu=%#RX64)\n",
3917	rc, pVM->fGlobalForcedActions, pVCpu->fLocalForcedActions));
3918	return rc;
3919	}
3920
3921	/* Pending VM request packets, such as hardware interrupts. */
3922	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
3923	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
3924	{
3925	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchVmReq);
3926	Log4Func(("Pending VM request forcing us back to ring-3\n"));
3927	return VINF_EM_PENDING_REQUEST;
3928	}
3929
3930	/* Pending PGM pool flushes. */
3931	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
3932	{
3933	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchPgmPoolFlush);
3934	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
3935	return VINF_PGM_POOL_FLUSH_PENDING;
3936	}
3937
3938	/* Pending DMA requests. */
3939	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
3940	{
3941	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchDma);
3942	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
3943	return VINF_EM_RAW_TO_R3;
3944	}
3945
3946	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3947	/*
3948	* Pending nested-guest events.
3949	*
3950	* Please note the priority of these events are specified and important.
3951	* See Intel spec. 29.4.3.2 "APIC-Write Emulation".
3952	* See Intel spec. 6.9 "Priority Among Simultaneous Exceptions And Interrupts".
3953	*/
3954	if (fIsNestedGuest)
3955	{
3956	/* Pending nested-guest APIC-write. */
3957	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
3958	{
3959	Log4Func(("Pending nested-guest APIC-write\n"));
3960	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitApicWrite(pVCpu);
3961	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
3962	return rcStrict;
3963	}
3964
3965	/* Pending nested-guest monitor-trap flag (MTF). */
3966	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF))
3967	{
3968	Log4Func(("Pending nested-guest MTF\n"));
3969	VBOXSTRICTRC rcStrict = IEMExecVmxVmexit(pVCpu, VMX_EXIT_MTF, 0 /* uExitQual */);
3970	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
3971	return rcStrict;
3972	}
3973
3974	/* Pending nested-guest VMX-preemption timer expired. */
3975	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER))
3976	{
3977	Log4Func(("Pending nested-guest preempt timer\n"));
3978	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitPreemptTimer(pVCpu);
3979	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
3980	return rcStrict;
3981	}
3982	}
3983	#else
3984	NOREF(fIsNestedGuest);
3985	#endif
3986
3987	return VINF_SUCCESS;
3988	}
3989
3990
3991	/**
3992	* Converts any TRPM trap into a pending HM event. This is typically used when
3993	* entering from ring-3 (not longjmp returns).
3994	*
3995	* @param pVCpu The cross context virtual CPU structure.
3996	*/
3997	static void vmxHCTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
3998	{
3999	Assert(TRPMHasTrap(pVCpu));
4000	Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending);
4001
4002	uint8_t uVector;
4003	TRPMEVENT enmTrpmEvent;
4004	uint32_t uErrCode;
4005	RTGCUINTPTR GCPtrFaultAddress;
4006	uint8_t cbInstr;
4007	bool fIcebp;
4008
4009	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, &fIcebp);
4010	AssertRC(rc);
4011
4012	uint32_t u32IntInfo;
4013	u32IntInfo = uVector \| VMX_IDT_VECTORING_INFO_VALID;
4014	u32IntInfo \|= HMTrpmEventTypeToVmxEventType(uVector, enmTrpmEvent, fIcebp);
4015
4016	rc = TRPMResetTrap(pVCpu);
4017	AssertRC(rc);
4018	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
4019	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
4020
4021	vmxHCSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
4022	}
4023
4024
4025	/**
4026	* Converts the pending HM event into a TRPM trap.
4027	*
4028	* @param pVCpu The cross context virtual CPU structure.
4029	*/
4030	static void vmxHCPendingEventToTrpmTrap(PVMCPUCC pVCpu)
4031	{
4032	Assert(VCPU_2_VMXSTATE(pVCpu).Event.fPending);
4033
4034	/* If a trap was already pending, we did something wrong! */
4035	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
4036
4037	uint32_t const u32IntInfo = VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo;
4038	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(u32IntInfo);
4039	TRPMEVENT const enmTrapType = HMVmxEventTypeToTrpmEventType(u32IntInfo);
4040
4041	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
4042
4043	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
4044	AssertRC(rc);
4045
4046	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
4047	TRPMSetErrorCode(pVCpu, VCPU_2_VMXSTATE(pVCpu).Event.u32ErrCode);
4048
4049	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(u32IntInfo))
4050	TRPMSetFaultAddress(pVCpu, VCPU_2_VMXSTATE(pVCpu).Event.GCPtrFaultAddress);
4051	else
4052	{
4053	uint8_t const uVectorType = VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo);
4054	switch (uVectorType)
4055	{
4056	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
4057	TRPMSetTrapDueToIcebp(pVCpu);
4058	RT_FALL_THRU();
4059	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
4060	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
4061	{
4062	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4063	\|\| ( uVector == X86_XCPT_BP /* INT3 */
4064	\|\| uVector == X86_XCPT_OF /* INTO */
4065	\|\| uVector == X86_XCPT_DB /* INT1 (ICEBP) */),
4066	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
4067	TRPMSetInstrLength(pVCpu, VCPU_2_VMXSTATE(pVCpu).Event.cbInstr);
4068	break;
4069	}
4070	}
4071	}
4072
4073	/* We're now done converting the pending event. */
4074	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false;
4075	}
4076
4077
4078	/**
4079	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
4080	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
4081	*
4082	* @param pVCpu The cross context virtual CPU structure.
4083	* @param pVmcsInfo The VMCS info. object.
4084	*/
4085	static void vmxHCSetIntWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
4086	{
4087	if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
4088	{
4089	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
4090	{
4091	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
4092	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
4093	AssertRC(rc);
4094	}
4095	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
4096	}
4097
4098
4099	/**
4100	* Clears the interrupt-window exiting control in the VMCS.
4101	*
4102	* @param pVCpu The cross context virtual CPU structure.
4103	* @param pVmcsInfo The VMCS info. object.
4104	*/
4105	DECLINLINE(void) vmxHCClearIntWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
4106	{
4107	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
4108	{
4109	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
4110	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
4111	AssertRC(rc);
4112	}
4113	}
4114
4115
4116	/**
4117	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
4118	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
4119	*
4120	* @param pVCpu The cross context virtual CPU structure.
4121	* @param pVmcsInfo The VMCS info. object.
4122	*/
4123	static void vmxHCSetNmiWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
4124	{
4125	if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
4126	{
4127	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
4128	{
4129	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
4130	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
4131	AssertRC(rc);
4132	Log4Func(("Setup NMI-window exiting\n"));
4133	}
4134	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
4135	}
4136
4137
4138	/**
4139	* Clears the NMI-window exiting control in the VMCS.
4140	*
4141	* @param pVCpu The cross context virtual CPU structure.
4142	* @param pVmcsInfo The VMCS info. object.
4143	*/
4144	DECLINLINE(void) vmxHCClearNmiWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
4145	{
4146	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
4147	{
4148	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
4149	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
4150	AssertRC(rc);
4151	}
4152	}
4153
4154
4155	/**
4156	* Injects an event into the guest upon VM-entry by updating the relevant fields
4157	* in the VM-entry area in the VMCS.
4158	*
4159	* @returns Strict VBox status code (i.e. informational status codes too).
4160	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
4161	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
4162	*
4163	* @param pVCpu The cross context virtual CPU structure.
4164	* @param pVmcsInfo The VMCS info object.
4165	* @param fIsNestedGuest Flag whether this is for a for a pending nested guest event.
4166	* @param pEvent The event being injected.
4167	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state. This
4168	* will be updated if necessary. This cannot not be NULL.
4169	* @param fStepping Whether we're single-stepping guest execution and should
4170	* return VINF_EM_DBG_STEPPED if the event is injected
4171	* directly (registers modified by us, not by hardware on
4172	* VM-entry).
4173	*/
4174	static VBOXSTRICTRC vmxHCInjectEventVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNestedGuest, PCHMEVENT pEvent,
4175	bool fStepping, uint32_t *pfIntrState)
4176	{
4177	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
4178	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
4179	Assert(pfIntrState);
4180
4181	#ifdef IN_NEM_DARWIN
4182	RT_NOREF(fIsNestedGuest, fStepping, pfIntrState);
4183	#endif
4184
4185	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4186	uint32_t u32IntInfo = pEvent->u64IntInfo;
4187	uint32_t const u32ErrCode = pEvent->u32ErrCode;
4188	uint32_t const cbInstr = pEvent->cbInstr;
4189	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
4190	uint8_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
4191	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
4192
4193	#ifdef VBOX_STRICT
4194	/*
4195	* Validate the error-code-valid bit for hardware exceptions.
4196	* No error codes for exceptions in real-mode.
4197	*
4198	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
4199	*/
4200	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
4201	&& !CPUMIsGuestInRealModeEx(pCtx))
4202	{
4203	switch (uVector)
4204	{
4205	case X86_XCPT_PF:
4206	case X86_XCPT_DF:
4207	case X86_XCPT_TS:
4208	case X86_XCPT_NP:
4209	case X86_XCPT_SS:
4210	case X86_XCPT_GP:
4211	case X86_XCPT_AC:
4212	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
4213	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
4214	RT_FALL_THRU();
4215	default:
4216	break;
4217	}
4218	}
4219
4220	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
4221	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
4222	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
4223	#endif
4224
4225	RT_NOREF(uVector);
4226	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
4227	\|\| uIntType == VMX_EXIT_INT_INFO_TYPE_NMI
4228	\|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT
4229	\|\| uIntType == VMX_EXIT_INT_INFO_TYPE_SW_XCPT)
4230	{
4231	Assert(uVector <= X86_XCPT_LAST);
4232	Assert(uIntType != VMX_EXIT_INT_INFO_TYPE_NMI \|\| uVector == X86_XCPT_NMI);
4233	Assert(uIntType != VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT \|\| uVector == X86_XCPT_DB);
4234	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).aStatInjectedXcpts[uVector]);
4235	}
4236	else
4237	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).aStatInjectedIrqs[uVector & MASK_INJECT_IRQ_STAT]);
4238
4239	/*
4240	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
4241	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
4242	* interrupt handler in the (real-mode) guest.
4243	*
4244	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
4245	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
4246	*/
4247	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
4248	{
4249	#ifndef IN_NEM_DARWIN
4250	if (pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fUnrestrictedGuest)
4251	#endif
4252	{
4253	/*
4254	* For CPUs with unrestricted guest execution enabled and with the guest
4255	* in real-mode, we must not set the deliver-error-code bit.
4256	*
4257	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
4258	*/
4259	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
4260	}
4261	#ifndef IN_NEM_DARWIN
4262	else
4263	{
4264	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4265	Assert(PDMVmmDevHeapIsEnabled(pVM));
4266	Assert(pVM->hm.s.vmx.pRealModeTSS);
4267	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
4268
4269	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
4270	int rc2 = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
4271	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
4272	AssertRCReturn(rc2, rc2);
4273
4274	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
4275	size_t const cbIdtEntry = sizeof(X86IDTR16);
4276	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
4277	{
4278	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
4279	if (uVector == X86_XCPT_DF)
4280	return VINF_EM_RESET;
4281
4282	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
4283	No error codes for exceptions in real-mode. */
4284	if (uVector == X86_XCPT_GP)
4285	{
4286	static HMEVENT const s_EventXcptDf
4287	= HMEVENT_INIT_ONLY_INT_INFO( RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
4288	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
4289	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
4290	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1));
4291	return vmxHCInjectEventVmcs(pVCpu, pVmcsInfo, fIsNestedGuest, &s_EventXcptDf, fStepping, pfIntrState);
4292	}
4293
4294	/*
4295	* If we're injecting an event with no valid IDT entry, inject a #GP.
4296	* No error codes for exceptions in real-mode.
4297	*
4298	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
4299	*/
4300	static HMEVENT const s_EventXcptGp
4301	= HMEVENT_INIT_ONLY_INT_INFO( RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
4302	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
4303	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
4304	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1));
4305	return vmxHCInjectEventVmcs(pVCpu, pVmcsInfo, fIsNestedGuest, &s_EventXcptGp, fStepping, pfIntrState);
4306	}
4307
4308	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
4309	uint16_t uGuestIp = pCtx->ip;
4310	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
4311	{
4312	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
4313	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
4314	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
4315	}
4316	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
4317	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
4318
4319	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
4320	X86IDTR16 IdtEntry;
4321	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
4322	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
4323	AssertRCReturn(rc2, rc2);
4324
4325	/* Construct the stack frame for the interrupt/exception handler. */
4326	VBOXSTRICTRC rcStrict;
4327	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
4328	if (rcStrict == VINF_SUCCESS)
4329	{
4330	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
4331	if (rcStrict == VINF_SUCCESS)
4332	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
4333	}
4334
4335	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
4336	if (rcStrict == VINF_SUCCESS)
4337	{
4338	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
4339	pCtx->rip = IdtEntry.offSel;
4340	pCtx->cs.Sel = IdtEntry.uSel;
4341	pCtx->cs.ValidSel = IdtEntry.uSel;
4342	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
4343	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
4344	&& uVector == X86_XCPT_PF)
4345	pCtx->cr2 = GCPtrFault;
4346
4347	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
4348	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
4349	\| HM_CHANGED_GUEST_RSP);
4350
4351	/*
4352	* If we delivered a hardware exception (other than an NMI) and if there was
4353	* block-by-STI in effect, we should clear it.
4354	*/
4355	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
4356	{
4357	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
4358	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
4359	Log4Func(("Clearing inhibition due to STI\n"));
4360	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4361	}
4362
4363	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
4364	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
4365
4366	/*
4367	* The event has been truly dispatched to the guest. Mark it as no longer pending so
4368	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
4369	*/
4370	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false;
4371
4372	/*
4373	* If we eventually support nested-guest execution without unrestricted guest execution,
4374	* we should set fInterceptEvents here.
4375	*/
4376	Assert(!fIsNestedGuest);
4377
4378	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
4379	if (fStepping)
4380	rcStrict = VINF_EM_DBG_STEPPED;
4381	}
4382	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
4383	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
4384	return rcStrict;
4385	}
4386	#else
4387	RT_NOREF(pVmcsInfo);
4388	#endif
4389	}
4390
4391	/*
4392	* Validate.
4393	*/
4394	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
4395	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
4396
4397	/*
4398	* Inject the event into the VMCS.
4399	*/
4400	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
4401	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
4402	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
4403	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
4404	AssertRC(rc);
4405
4406	/*
4407	* Update guest CR2 if this is a page-fault.
4408	*/
4409	if (VMX_ENTRY_INT_INFO_IS_XCPT_PF(u32IntInfo))
4410	pCtx->cr2 = GCPtrFault;
4411
4412	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
4413	return VINF_SUCCESS;
4414	}
4415
4416
4417	/**
4418	* Evaluates the event to be delivered to the guest and sets it as the pending
4419	* event.
4420	*
4421	* Toggling of interrupt force-flags here is safe since we update TRPM on premature
4422	* exits to ring-3 before executing guest code, see vmxHCExitToRing3(). We must
4423	* NOT restore these force-flags.
4424	*
4425	* @returns Strict VBox status code (i.e. informational status codes too).
4426	* @param pVCpu The cross context virtual CPU structure.
4427	* @param pVmcsInfo The VMCS information structure.
4428	* @param fIsNestedGuest Flag whether the evaluation happens for a nested guest.
4429	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
4430	*/
4431	static VBOXSTRICTRC vmxHCEvaluatePendingEvent(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNestedGuest, uint32_t *pfIntrState)
4432	{
4433	Assert(pfIntrState);
4434	Assert(!TRPMHasTrap(pVCpu));
4435
4436	/*
4437	* Compute/update guest-interruptibility state related FFs.
4438	* The FFs will be used below while evaluating events to be injected.
4439	*/
4440	*pfIntrState = vmxHCGetGuestIntrStateAndUpdateFFs(pVCpu);
4441
4442	/*
4443	* Evaluate if a new event needs to be injected.
4444	* An event that's already pending has already performed all necessary checks.
4445	*/
4446	if ( !VCPU_2_VMXSTATE(pVCpu).Event.fPending
4447	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4448	{
4449	/** @todo SMI. SMIs take priority over NMIs. */
4450
4451	/*
4452	* NMIs.
4453	* NMIs take priority over external interrupts.
4454	*/
4455	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4456	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4457	#endif
4458	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
4459	{
4460	/*
4461	* For a guest, the FF always indicates the guest's ability to receive an NMI.
4462	*
4463	* For a nested-guest, the FF always indicates the outer guest's ability to
4464	* receive an NMI while the guest-interruptibility state bit depends on whether
4465	* the nested-hypervisor is using virtual-NMIs.
4466	*/
4467	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
4468	{
4469	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4470	if ( fIsNestedGuest
4471	&& CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_NMI_EXIT))
4472	return IEMExecVmxVmexitXcptNmi(pVCpu);
4473	#endif
4474	vmxHCSetPendingXcptNmi(pVCpu);
4475	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
4476	Log4Func(("NMI pending injection\n"));
4477
4478	/* We've injected the NMI, bail. */
4479	return VINF_SUCCESS;
4480	}
4481	else if (!fIsNestedGuest)
4482	vmxHCSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
4483	}
4484
4485	/*
4486	* External interrupts (PIC/APIC).
4487	* Once PDMGetInterrupt() returns a valid interrupt we -must- deliver it.
4488	* We cannot re-request the interrupt from the controller again.
4489	*/
4490	if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
4491	&& !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
4492	{
4493	Assert(!DBGFIsStepping(pVCpu));
4494	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
4495	AssertRC(rc);
4496
4497	/*
4498	* We must not check EFLAGS directly when executing a nested-guest, use
4499	* CPUMIsGuestPhysIntrEnabled() instead as EFLAGS.IF does not control the blocking of
4500	* external interrupts when "External interrupt exiting" is set. This fixes a nasty
4501	* SMP hang while executing nested-guest VCPUs on spinlocks which aren't rescued by
4502	* other VM-exits (like a preemption timer), see @bugref{9562#c18}.
4503	*
4504	* See Intel spec. 25.4.1 "Event Blocking".
4505	*/
4506	if (CPUMIsGuestPhysIntrEnabled(pVCpu))
4507	{
4508	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4509	if ( fIsNestedGuest
4510	&& CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
4511	{
4512	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0 /* uVector /, true / fIntPending */);
4513	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
4514	return rcStrict;
4515	}
4516	#endif
4517	uint8_t u8Interrupt;
4518	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
4519	if (RT_SUCCESS(rc))
4520	{
4521	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4522	if ( fIsNestedGuest
4523	&& CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
4524	{
4525	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
4526	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
4527	return rcStrict;
4528	}
4529	#endif
4530	vmxHCSetPendingExtInt(pVCpu, u8Interrupt);
4531	Log4Func(("External interrupt (%#x) pending injection\n", u8Interrupt));
4532	}
4533	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
4534	{
4535	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchTprMaskedIrq);
4536
4537	if ( !fIsNestedGuest
4538	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
4539	vmxHCApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
4540	/* else: for nested-guests, TPR threshold is picked up while merging VMCS controls. */
4541
4542	/*
4543	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
4544	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
4545	* need to re-set this force-flag here.
4546	*/
4547	}
4548	else
4549	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchGuestIrq);
4550
4551	/* We've injected the interrupt or taken necessary action, bail. */
4552	return VINF_SUCCESS;
4553	}
4554	if (!fIsNestedGuest)
4555	vmxHCSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
4556	}
4557	}
4558	else if (!fIsNestedGuest)
4559	{
4560	/*
4561	* An event is being injected or we are in an interrupt shadow. Check if another event is
4562	* pending. If so, instruct VT-x to cause a VM-exit as soon as the guest is ready to accept
4563	* the pending event.
4564	*/
4565	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
4566	vmxHCSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
4567	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
4568	&& !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
4569	vmxHCSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
4570	}
4571	/* else: for nested-guests, NMI/interrupt-window exiting will be picked up when merging VMCS controls. */
4572
4573	return VINF_SUCCESS;
4574	}
4575
4576
4577	/**
4578	* Injects any pending events into the guest if the guest is in a state to
4579	* receive them.
4580	*
4581	* @returns Strict VBox status code (i.e. informational status codes too).
4582	* @param pVCpu The cross context virtual CPU structure.
4583	* @param pVmcsInfo The VMCS information structure.
4584	* @param fIsNestedGuest Flag whether the event injection happens for a nested guest.
4585	* @param fIntrState The VT-x guest-interruptibility state.
4586	* @param fStepping Whether we are single-stepping the guest using the
4587	* hypervisor debugger and should return
4588	* VINF_EM_DBG_STEPPED if the event was dispatched
4589	* directly.
4590	*/
4591	static VBOXSTRICTRC vmxHCInjectPendingEvent(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNestedGuest,
4592	uint32_t fIntrState, bool fStepping)
4593	{
4594	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
4595	#ifndef IN_NEM_DARWIN
4596	Assert(VMMRZCallRing3IsEnabled(pVCpu));
4597	#endif
4598
4599	#ifdef VBOX_STRICT
4600	/*
4601	* Verify guest-interruptibility state.
4602	*
4603	* We put this in a scoped block so we do not accidentally use fBlockSti or fBlockMovSS,
4604	* since injecting an event may modify the interruptibility state and we must thus always
4605	* use fIntrState.
4606	*/
4607	{
4608	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
4609	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
4610	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
4611	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
4612	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
4613	Assert(!TRPMHasTrap(pVCpu));
4614	NOREF(fBlockMovSS); NOREF(fBlockSti);
4615	}
4616	#endif
4617
4618	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
4619	if (VCPU_2_VMXSTATE(pVCpu).Event.fPending)
4620	{
4621	/*
4622	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
4623	* pending even while injecting an event and in this case, we want a VM-exit as soon as
4624	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
4625	*
4626	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
4627	*/
4628	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo);
4629	#ifdef VBOX_STRICT
4630	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
4631	{
4632	Assert(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
4633	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI));
4634	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
4635	}
4636	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
4637	{
4638	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI));
4639	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI));
4640	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
4641	}
4642	#endif
4643	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
4644	uIntType));
4645
4646	/*
4647	* Inject the event and get any changes to the guest-interruptibility state.
4648	*
4649	* The guest-interruptibility state may need to be updated if we inject the event
4650	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
4651	*/
4652	rcStrict = vmxHCInjectEventVmcs(pVCpu, pVmcsInfo, fIsNestedGuest, &VCPU_2_VMXSTATE(pVCpu).Event, fStepping, &fIntrState);
4653	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
4654
4655	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
4656	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectInterrupt);
4657	else
4658	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectXcpt);
4659	}
4660
4661	/*
4662	* Deliver any pending debug exceptions if the guest is single-stepping using EFLAGS.TF and
4663	* is an interrupt shadow (block-by-STI or block-by-MOV SS).
4664	*/
4665	if ( (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
4666	&& !fIsNestedGuest)
4667	{
4668	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
4669
4670	if (!VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
4671	{
4672	/*
4673	* Set or clear the BS bit depending on whether the trap flag is active or not. We need
4674	* to do both since we clear the BS bit from the VMCS while exiting to ring-3.
4675	*/
4676	Assert(!DBGFIsStepping(pVCpu));
4677	uint8_t const fTrapFlag = !!(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_TF);
4678	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, fTrapFlag << VMX_BF_VMCS_PENDING_DBG_XCPT_BS_SHIFT);
4679	AssertRC(rc);
4680	}
4681	else
4682	{
4683	/*
4684	* We must not deliver a debug exception when single-stepping over STI/Mov-SS in the
4685	* hypervisor debugger using EFLAGS.TF but rather clear interrupt inhibition. However,
4686	* we take care of this case in vmxHCExportSharedDebugState and also the case if
4687	* we use MTF, so just make sure it's called before executing guest-code.
4688	*/
4689	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_DR_MASK);
4690	}
4691	}
4692	/* else: for nested-guest currently handling while merging controls. */
4693
4694	/*
4695	* Finally, update the guest-interruptibility state.
4696	*
4697	* This is required for the real-on-v86 software interrupt injection, for
4698	* pending debug exceptions as well as updates to the guest state from ring-3 (IEM).
4699	*/
4700	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, fIntrState);
4701	AssertRC(rc);
4702
4703	/*
4704	* There's no need to clear the VM-entry interruption-information field here if we're not
4705	* injecting anything. VT-x clears the valid bit on every VM-exit.
4706	*
4707	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
4708	*/
4709
4710	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
4711	return rcStrict;
4712	}
4713
4714
4715	/**
4716	* Tries to determine what part of the guest-state VT-x has deemed as invalid
4717	* and update error record fields accordingly.
4718	*
4719	* @returns VMX_IGS_* error codes.
4720	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
4721	* wrong with the guest state.
4722	*
4723	* @param pVCpu The cross context virtual CPU structure.
4724	* @param pVmcsInfo The VMCS info. object.
4725	*
4726	* @remarks This function assumes our cache of the VMCS controls
4727	* are valid, i.e. vmxHCCheckCachedVmcsCtls() succeeded.
4728	*/
4729	static uint32_t vmxHCCheckGuestState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
4730	{
4731	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
4732	#define HMVMX_CHECK_BREAK(expr, err) do { \
4733	if (!(expr)) { uError = (err); break; } \
4734	} while (0)
4735
4736	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4737	uint32_t uError = VMX_IGS_ERROR;
4738	uint32_t u32IntrState = 0;
4739	#ifndef IN_NEM_DARWIN
4740	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4741	bool const fUnrestrictedGuest = VM_IS_VMX_UNRESTRICTED_GUEST(pVM);
4742	#else
4743	bool const fUnrestrictedGuest = true;
4744	#endif
4745	do
4746	{
4747	int rc;
4748
4749	/*
4750	* Guest-interruptibility state.
4751	*
4752	* Read this first so that any check that fails prior to those that actually
4753	* require the guest-interruptibility state would still reflect the correct
4754	* VMCS value and avoids causing further confusion.
4755	*/
4756	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
4757	AssertRC(rc);
4758
4759	uint32_t u32Val;
4760	uint64_t u64Val;
4761
4762	/*
4763	* CR0.
4764	*/
4765	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
4766	uint64_t fSetCr0 = (g_HmMsrs.u.vmx.u64Cr0Fixed0 & g_HmMsrs.u.vmx.u64Cr0Fixed1);
4767	uint64_t const fZapCr0 = (g_HmMsrs.u.vmx.u64Cr0Fixed0 \| g_HmMsrs.u.vmx.u64Cr0Fixed1);
4768	/* Exceptions for unrestricted guest execution for CR0 fixed bits (PE, PG).
4769	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
4770	if (fUnrestrictedGuest)
4771	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
4772
4773	uint64_t u64GuestCr0;
4774	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0, &u64GuestCr0);
4775	AssertRC(rc);
4776	HMVMX_CHECK_BREAK((u64GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
4777	HMVMX_CHECK_BREAK(!(u64GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
4778	if ( !fUnrestrictedGuest
4779	&& (u64GuestCr0 & X86_CR0_PG)
4780	&& !(u64GuestCr0 & X86_CR0_PE))
4781	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
4782
4783	/*
4784	* CR4.
4785	*/
4786	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
4787	uint64_t const fSetCr4 = (g_HmMsrs.u.vmx.u64Cr4Fixed0 & g_HmMsrs.u.vmx.u64Cr4Fixed1);
4788	uint64_t const fZapCr4 = (g_HmMsrs.u.vmx.u64Cr4Fixed0 \| g_HmMsrs.u.vmx.u64Cr4Fixed1);
4789
4790	uint64_t u64GuestCr4;
4791	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR4, &u64GuestCr4);
4792	AssertRC(rc);
4793	HMVMX_CHECK_BREAK((u64GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
4794	HMVMX_CHECK_BREAK(!(u64GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
4795
4796	/*
4797	* IA32_DEBUGCTL MSR.
4798	*/
4799	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
4800	AssertRC(rc);
4801	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
4802	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
4803	{
4804	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
4805	}
4806	uint64_t u64DebugCtlMsr = u64Val;
4807
4808	#ifdef VBOX_STRICT
4809	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY, &u32Val);
4810	AssertRC(rc);
4811	Assert(u32Val == pVmcsInfo->u32EntryCtls);
4812	#endif
4813	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
4814
4815	/*
4816	* RIP and RFLAGS.
4817	*/
4818	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RIP, &u64Val);
4819	AssertRC(rc);
4820	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
4821	if ( !fLongModeGuest
4822	\|\| !pCtx->cs.Attr.n.u1Long)
4823	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
4824	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
4825	* must be identical if the "IA-32e mode guest" VM-entry
4826	* control is 1 and CS.L is 1. No check applies if the
4827	* CPU supports 64 linear-address bits. */
4828
4829	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
4830	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RFLAGS, &u64Val);
4831	AssertRC(rc);
4832	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
4833	VMX_IGS_RFLAGS_RESERVED);
4834	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
4835	uint32_t const u32Eflags = u64Val;
4836
4837	if ( fLongModeGuest
4838	\|\| ( fUnrestrictedGuest
4839	&& !(u64GuestCr0 & X86_CR0_PE)))
4840	{
4841	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
4842	}
4843
4844	uint32_t u32EntryInfo;
4845	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
4846	AssertRC(rc);
4847	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
4848	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
4849
4850	/*
4851	* 64-bit checks.
4852	*/
4853	if (fLongModeGuest)
4854	{
4855	HMVMX_CHECK_BREAK(u64GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
4856	HMVMX_CHECK_BREAK(u64GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
4857	}
4858
4859	if ( !fLongModeGuest
4860	&& (u64GuestCr4 & X86_CR4_PCIDE))
4861	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
4862
4863	/** @todo CR3 field must be such that bits 63:52 and bits in the range
4864	* 51:32 beyond the processor's physical-address width are 0. */
4865
4866	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
4867	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
4868	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
4869
4870	#ifndef IN_NEM_DARWIN
4871	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
4872	AssertRC(rc);
4873	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
4874
4875	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
4876	AssertRC(rc);
4877	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
4878	#endif
4879
4880	/*
4881	* PERF_GLOBAL MSR.
4882	*/
4883	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
4884	{
4885	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
4886	AssertRC(rc);
4887	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
4888	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
4889	}
4890
4891	/*
4892	* PAT MSR.
4893	*/
4894	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
4895	{
4896	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
4897	AssertRC(rc);
4898	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
4899	for (unsigned i = 0; i < 8; i++)
4900	{
4901	uint8_t u8Val = (u64Val & 0xff);
4902	if ( u8Val != 0 /* UC */
4903	&& u8Val != 1 /* WC */
4904	&& u8Val != 4 /* WT */
4905	&& u8Val != 5 /* WP */
4906	&& u8Val != 6 /* WB */
4907	&& u8Val != 7 /* UC- */)
4908	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
4909	u64Val >>= 8;
4910	}
4911	}
4912
4913	/*
4914	* EFER MSR.
4915	*/
4916	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
4917	{
4918	Assert(g_fHmVmxSupportsVmcsEfer);
4919	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
4920	AssertRC(rc);
4921	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
4922	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
4923	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
4924	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
4925	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
4926	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
4927	* iemVmxVmentryCheckGuestState(). */
4928	HMVMX_CHECK_BREAK( fUnrestrictedGuest
4929	\|\| !(u64GuestCr0 & X86_CR0_PG)
4930	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
4931	VMX_IGS_EFER_LMA_LME_MISMATCH);
4932	}
4933
4934	/*
4935	* Segment registers.
4936	*/
4937	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
4938	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
4939	if (!(u32Eflags & X86_EFL_VM))
4940	{
4941	/* CS */
4942	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
4943	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
4944	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
4945	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4946	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
4947	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
4948	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
4949	/* CS cannot be loaded with NULL in protected mode. */
4950	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
4951	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
4952	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4953	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
4954	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4955	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
4956	else if (fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
4957	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
4958	else
4959	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
4960
4961	/* SS */
4962	HMVMX_CHECK_BREAK( fUnrestrictedGuest
4963	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
4964	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
4965	if ( !(pCtx->cr0 & X86_CR0_PE)
4966	\|\| pCtx->cs.Attr.n.u4Type == 3)
4967	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
4968
4969	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4970	{
4971	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
4972	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
4973	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
4974	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
4975	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4976	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
4977	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
4978	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
4979	}
4980
4981	/* DS, ES, FS, GS - only check for usable selectors, see vmxHCExportGuestSReg(). */
4982	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4983	{
4984	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
4985	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
4986	HMVMX_CHECK_BREAK( fUnrestrictedGuest
4987	\|\| pCtx->ds.Attr.n.u4Type > 11
4988	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
4989	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
4990	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
4991	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4992	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
4993	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
4994	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
4995	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4996	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
4997	}
4998	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4999	{
5000	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
5001	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
5002	HMVMX_CHECK_BREAK( fUnrestrictedGuest
5003	\|\| pCtx->es.Attr.n.u4Type > 11
5004	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
5005	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
5006	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
5007	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
5008	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
5009	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
5010	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
5011	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5012	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
5013	}
5014	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
5015	{
5016	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
5017	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
5018	HMVMX_CHECK_BREAK( fUnrestrictedGuest
5019	\|\| pCtx->fs.Attr.n.u4Type > 11
5020	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
5021	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
5022	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
5023	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
5024	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
5025	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
5026	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
5027	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5028	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
5029	}
5030	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
5031	{
5032	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
5033	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
5034	HMVMX_CHECK_BREAK( fUnrestrictedGuest
5035	\|\| pCtx->gs.Attr.n.u4Type > 11
5036	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
5037	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
5038	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
5039	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
5040	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
5041	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
5042	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
5043	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5044	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
5045	}
5046	/* 64-bit capable CPUs. */
5047	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
5048	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
5049	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
5050	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
5051	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
5052	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
5053	VMX_IGS_LONGMODE_SS_BASE_INVALID);
5054	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
5055	VMX_IGS_LONGMODE_DS_BASE_INVALID);
5056	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
5057	VMX_IGS_LONGMODE_ES_BASE_INVALID);
5058	}
5059	else
5060	{
5061	/* V86 mode checks. */
5062	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
5063	if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
5064	{
5065	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
5066	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
5067	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
5068	}
5069	else
5070	{
5071	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
5072	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
5073	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
5074	}
5075
5076	/* CS */
5077	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
5078	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
5079	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
5080	/* SS */
5081	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
5082	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
5083	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
5084	/* DS */
5085	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
5086	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
5087	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
5088	/* ES */
5089	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
5090	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
5091	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
5092	/* FS */
5093	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
5094	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
5095	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
5096	/* GS */
5097	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
5098	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
5099	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
5100	/* 64-bit capable CPUs. */
5101	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
5102	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
5103	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
5104	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
5105	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
5106	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
5107	VMX_IGS_LONGMODE_SS_BASE_INVALID);
5108	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
5109	VMX_IGS_LONGMODE_DS_BASE_INVALID);
5110	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
5111	VMX_IGS_LONGMODE_ES_BASE_INVALID);
5112	}
5113
5114	/*
5115	* TR.
5116	*/
5117	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
5118	/* 64-bit capable CPUs. */
5119	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
5120	if (fLongModeGuest)
5121	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
5122	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
5123	else
5124	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
5125	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
5126	VMX_IGS_TR_ATTR_TYPE_INVALID);
5127	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
5128	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
5129	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
5130	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
5131	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
5132	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
5133	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
5134	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
5135
5136	/*
5137	* GDTR and IDTR (64-bit capable checks).
5138	*/
5139	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
5140	AssertRC(rc);
5141	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
5142
5143	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
5144	AssertRC(rc);
5145	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
5146
5147	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
5148	AssertRC(rc);
5149	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
5150
5151	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
5152	AssertRC(rc);
5153	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
5154
5155	/*
5156	* Guest Non-Register State.
5157	*/
5158	/* Activity State. */
5159	uint32_t u32ActivityState;
5160	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
5161	AssertRC(rc);
5162	HMVMX_CHECK_BREAK( !u32ActivityState
5163	\|\| (u32ActivityState & RT_BF_GET(g_HmMsrs.u.vmx.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
5164	VMX_IGS_ACTIVITY_STATE_INVALID);
5165	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
5166	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
5167
5168	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
5169	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
5170	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
5171
5172	/** @todo Activity state and injecting interrupts. Left as a todo since we
5173	* currently don't use activity states but ACTIVE. */
5174
5175	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
5176	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
5177
5178	/* Guest interruptibility-state. */
5179	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
5180	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5181	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
5182	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
5183	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
5184	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
5185	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
5186	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
5187	{
5188	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
5189	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
5190	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
5191	}
5192	else if (VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
5193	{
5194	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
5195	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
5196	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
5197	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
5198	}
5199	/** @todo Assumes the processor is not in SMM. */
5200	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
5201	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
5202	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
5203	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
5204	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
5205	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
5206	&& VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
5207	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI), VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
5208
5209	/* Pending debug exceptions. */
5210	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
5211	AssertRC(rc);
5212	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
5213	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
5214	u32Val = u64Val; /* For pending debug exceptions checks below. */
5215
5216	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
5217	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
5218	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
5219	{
5220	if ( (u32Eflags & X86_EFL_TF)
5221	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
5222	{
5223	/* Bit 14 is PendingDebug.BS. */
5224	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
5225	}
5226	if ( !(u32Eflags & X86_EFL_TF)
5227	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
5228	{
5229	/* Bit 14 is PendingDebug.BS. */
5230	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
5231	}
5232	}
5233
5234	#ifndef IN_NEM_DARWIN
5235	/* VMCS link pointer. */
5236	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
5237	AssertRC(rc);
5238	if (u64Val != UINT64_C(0xffffffffffffffff))
5239	{
5240	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
5241	/** @todo Bits beyond the processor's physical-address width MBZ. */
5242	/** @todo SMM checks. */
5243	Assert(pVmcsInfo->HCPhysShadowVmcs == u64Val);
5244	Assert(pVmcsInfo->pvShadowVmcs);
5245	VMXVMCSREVID VmcsRevId;
5246	VmcsRevId.u = (uint32_t )pVmcsInfo->pvShadowVmcs;
5247	HMVMX_CHECK_BREAK(VmcsRevId.n.u31RevisionId == RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_ID),
5248	VMX_IGS_VMCS_LINK_PTR_SHADOW_VMCS_ID_INVALID);
5249	HMVMX_CHECK_BREAK(VmcsRevId.n.fIsShadowVmcs == (uint32_t)!!(pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING),
5250	VMX_IGS_VMCS_LINK_PTR_NOT_SHADOW);
5251	}
5252
5253	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
5254	* not using nested paging? */
5255	if ( VM_IS_VMX_NESTED_PAGING(pVM)
5256	&& !fLongModeGuest
5257	&& CPUMIsGuestInPAEModeEx(pCtx))
5258	{
5259	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
5260	AssertRC(rc);
5261	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
5262
5263	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
5264	AssertRC(rc);
5265	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
5266
5267	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
5268	AssertRC(rc);
5269	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
5270
5271	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
5272	AssertRC(rc);
5273	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
5274	}
5275	#endif
5276
5277	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
5278	if (uError == VMX_IGS_ERROR)
5279	uError = VMX_IGS_REASON_NOT_FOUND;
5280	} while (0);
5281
5282	VCPU_2_VMXSTATE(pVCpu).u32HMError = uError;
5283	VCPU_2_VMXSTATE(pVCpu).vmx.LastError.u32GuestIntrState = u32IntrState;
5284	return uError;
5285
5286	#undef HMVMX_ERROR_BREAK
5287	#undef HMVMX_CHECK_BREAK
5288	}
5289
5290
5291	#ifndef HMVMX_USE_FUNCTION_TABLE
5292	/**
5293	* Handles a guest VM-exit from hardware-assisted VMX execution.
5294	*
5295	* @returns Strict VBox status code (i.e. informational status codes too).
5296	* @param pVCpu The cross context virtual CPU structure.
5297	* @param pVmxTransient The VMX-transient structure.
5298	*/
5299	DECLINLINE(VBOXSTRICTRC) vmxHCHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5300	{
5301	#ifdef DEBUG_ramshankar
5302	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
5303	do { \
5304	if (a_fSave != 0) \
5305	vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL); \
5306	VBOXSTRICTRC rcStrict = a_CallExpr; \
5307	if (a_fSave != 0) \
5308	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST); \
5309	return rcStrict; \
5310	} while (0)
5311	#else
5312	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
5313	#endif
5314	uint32_t const uExitReason = pVmxTransient->uExitReason;
5315	switch (uExitReason)
5316	{
5317	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, vmxHCExitEptMisconfig(pVCpu, pVmxTransient));
5318	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, vmxHCExitEptViolation(pVCpu, pVmxTransient));
5319	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, vmxHCExitIoInstr(pVCpu, pVmxTransient));
5320	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, vmxHCExitCpuid(pVCpu, pVmxTransient));
5321	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, vmxHCExitRdtsc(pVCpu, pVmxTransient));
5322	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, vmxHCExitRdtscp(pVCpu, pVmxTransient));
5323	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, vmxHCExitApicAccess(pVCpu, pVmxTransient));
5324	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, vmxHCExitXcptOrNmi(pVCpu, pVmxTransient));
5325	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, vmxHCExitMovCRx(pVCpu, pVmxTransient));
5326	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, vmxHCExitExtInt(pVCpu, pVmxTransient));
5327	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, vmxHCExitIntWindow(pVCpu, pVmxTransient));
5328	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, vmxHCExitTprBelowThreshold(pVCpu, pVmxTransient));
5329	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, vmxHCExitMwait(pVCpu, pVmxTransient));
5330	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, vmxHCExitMonitor(pVCpu, pVmxTransient));
5331	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, vmxHCExitTaskSwitch(pVCpu, pVmxTransient));
5332	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, vmxHCExitPreemptTimer(pVCpu, pVmxTransient));
5333	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, vmxHCExitRdmsr(pVCpu, pVmxTransient));
5334	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, vmxHCExitWrmsr(pVCpu, pVmxTransient));
5335	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, vmxHCExitVmcall(pVCpu, pVmxTransient));
5336	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, vmxHCExitMovDRx(pVCpu, pVmxTransient));
5337	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, vmxHCExitHlt(pVCpu, pVmxTransient));
5338	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, vmxHCExitInvd(pVCpu, pVmxTransient));
5339	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, vmxHCExitInvlpg(pVCpu, pVmxTransient));
5340	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, vmxHCExitMtf(pVCpu, pVmxTransient));
5341	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, vmxHCExitPause(pVCpu, pVmxTransient));
5342	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, vmxHCExitWbinvd(pVCpu, pVmxTransient));
5343	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, vmxHCExitXsetbv(pVCpu, pVmxTransient));
5344	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, vmxHCExitInvpcid(pVCpu, pVmxTransient));
5345	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, vmxHCExitGetsec(pVCpu, pVmxTransient));
5346	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, vmxHCExitRdpmc(pVCpu, pVmxTransient));
5347	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5348	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, vmxHCExitVmclear(pVCpu, pVmxTransient));
5349	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, vmxHCExitVmlaunch(pVCpu, pVmxTransient));
5350	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, vmxHCExitVmptrld(pVCpu, pVmxTransient));
5351	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, vmxHCExitVmptrst(pVCpu, pVmxTransient));
5352	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, vmxHCExitVmread(pVCpu, pVmxTransient));
5353	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, vmxHCExitVmwrite(pVCpu, pVmxTransient));
5354	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, vmxHCExitVmresume(pVCpu, pVmxTransient));
5355	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, vmxHCExitVmxoff(pVCpu, pVmxTransient));
5356	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, vmxHCExitVmxon(pVCpu, pVmxTransient));
5357	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, vmxHCExitInvvpid(pVCpu, pVmxTransient));
5358	#else
5359	case VMX_EXIT_VMCLEAR:
5360	case VMX_EXIT_VMLAUNCH:
5361	case VMX_EXIT_VMPTRLD:
5362	case VMX_EXIT_VMPTRST:
5363	case VMX_EXIT_VMREAD:
5364	case VMX_EXIT_VMRESUME:
5365	case VMX_EXIT_VMWRITE:
5366	case VMX_EXIT_VMXOFF:
5367	case VMX_EXIT_VMXON:
5368	case VMX_EXIT_INVVPID:
5369	return vmxHCExitSetPendingXcptUD(pVCpu, pVmxTransient);
5370	#endif
5371	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5372	case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, vmxHCExitInvept(pVCpu, pVmxTransient));
5373	#else
5374	case VMX_EXIT_INVEPT: return vmxHCExitSetPendingXcptUD(pVCpu, pVmxTransient);
5375	#endif
5376
5377	case VMX_EXIT_TRIPLE_FAULT: return vmxHCExitTripleFault(pVCpu, pVmxTransient);
5378	case VMX_EXIT_NMI_WINDOW: return vmxHCExitNmiWindow(pVCpu, pVmxTransient);
5379	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return vmxHCExitErrInvalidGuestState(pVCpu, pVmxTransient);
5380
5381	case VMX_EXIT_INIT_SIGNAL:
5382	case VMX_EXIT_SIPI:
5383	case VMX_EXIT_IO_SMI:
5384	case VMX_EXIT_SMI:
5385	case VMX_EXIT_ERR_MSR_LOAD:
5386	case VMX_EXIT_ERR_MACHINE_CHECK:
5387	case VMX_EXIT_PML_FULL:
5388	case VMX_EXIT_VIRTUALIZED_EOI:
5389	case VMX_EXIT_GDTR_IDTR_ACCESS:
5390	case VMX_EXIT_LDTR_TR_ACCESS:
5391	case VMX_EXIT_APIC_WRITE:
5392	case VMX_EXIT_RDRAND:
5393	case VMX_EXIT_RSM:
5394	case VMX_EXIT_VMFUNC:
5395	case VMX_EXIT_ENCLS:
5396	case VMX_EXIT_RDSEED:
5397	case VMX_EXIT_XSAVES:
5398	case VMX_EXIT_XRSTORS:
5399	case VMX_EXIT_UMWAIT:
5400	case VMX_EXIT_TPAUSE:
5401	case VMX_EXIT_LOADIWKEY:
5402	default:
5403	return vmxHCExitErrUnexpected(pVCpu, pVmxTransient);
5404	}
5405	#undef VMEXIT_CALL_RET
5406	}
5407	#endif /* !HMVMX_USE_FUNCTION_TABLE */
5408
5409
5410	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5411	/**
5412	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
5413	*
5414	* @returns Strict VBox status code (i.e. informational status codes too).
5415	* @param pVCpu The cross context virtual CPU structure.
5416	* @param pVmxTransient The VMX-transient structure.
5417	*/
5418	DECLINLINE(VBOXSTRICTRC) vmxHCHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5419	{
5420	uint32_t const uExitReason = pVmxTransient->uExitReason;
5421	switch (uExitReason)
5422	{
5423	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5424	case VMX_EXIT_EPT_MISCONFIG: return vmxHCExitEptMisconfigNested(pVCpu, pVmxTransient);
5425	case VMX_EXIT_EPT_VIOLATION: return vmxHCExitEptViolationNested(pVCpu, pVmxTransient);
5426	# else
5427	case VMX_EXIT_EPT_MISCONFIG: return vmxHCExitEptMisconfig(pVCpu, pVmxTransient);
5428	case VMX_EXIT_EPT_VIOLATION: return vmxHCExitEptViolation(pVCpu, pVmxTransient);
5429	# endif
5430	case VMX_EXIT_XCPT_OR_NMI: return vmxHCExitXcptOrNmiNested(pVCpu, pVmxTransient);
5431	case VMX_EXIT_IO_INSTR: return vmxHCExitIoInstrNested(pVCpu, pVmxTransient);
5432	case VMX_EXIT_HLT: return vmxHCExitHltNested(pVCpu, pVmxTransient);
5433
5434	/*
5435	* We shouldn't direct host physical interrupts to the nested-guest.
5436	*/
5437	case VMX_EXIT_EXT_INT:
5438	return vmxHCExitExtInt(pVCpu, pVmxTransient);
5439
5440	/*
5441	* Instructions that cause VM-exits unconditionally or the condition is
5442	* always taken solely from the nested hypervisor (meaning if the VM-exit
5443	* happens, it's guaranteed to be a nested-guest VM-exit).
5444	*
5445	* - Provides VM-exit instruction length ONLY.
5446	*/
5447	case VMX_EXIT_CPUID: /* Unconditional. */
5448	case VMX_EXIT_VMCALL:
5449	case VMX_EXIT_GETSEC:
5450	case VMX_EXIT_INVD:
5451	case VMX_EXIT_XSETBV:
5452	case VMX_EXIT_VMLAUNCH:
5453	case VMX_EXIT_VMRESUME:
5454	case VMX_EXIT_VMXOFF:
5455	case VMX_EXIT_ENCLS: /* Condition specified solely by nested hypervisor. */
5456	case VMX_EXIT_VMFUNC:
5457	return vmxHCExitInstrNested(pVCpu, pVmxTransient);
5458
5459	/*
5460	* Instructions that cause VM-exits unconditionally or the condition is
5461	* always taken solely from the nested hypervisor (meaning if the VM-exit
5462	* happens, it's guaranteed to be a nested-guest VM-exit).
5463	*
5464	* - Provides VM-exit instruction length.
5465	* - Provides VM-exit information.
5466	* - Optionally provides Exit qualification.
5467	*
5468	* Since Exit qualification is 0 for all VM-exits where it is not
5469	* applicable, reading and passing it to the guest should produce
5470	* defined behavior.
5471	*
5472	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
5473	*/
5474	case VMX_EXIT_INVEPT: /* Unconditional. */
5475	case VMX_EXIT_INVVPID:
5476	case VMX_EXIT_VMCLEAR:
5477	case VMX_EXIT_VMPTRLD:
5478	case VMX_EXIT_VMPTRST:
5479	case VMX_EXIT_VMXON:
5480	case VMX_EXIT_GDTR_IDTR_ACCESS: /* Condition specified solely by nested hypervisor. */
5481	case VMX_EXIT_LDTR_TR_ACCESS:
5482	case VMX_EXIT_RDRAND:
5483	case VMX_EXIT_RDSEED:
5484	case VMX_EXIT_XSAVES:
5485	case VMX_EXIT_XRSTORS:
5486	case VMX_EXIT_UMWAIT:
5487	case VMX_EXIT_TPAUSE:
5488	return vmxHCExitInstrWithInfoNested(pVCpu, pVmxTransient);
5489
5490	case VMX_EXIT_RDTSC: return vmxHCExitRdtscNested(pVCpu, pVmxTransient);
5491	case VMX_EXIT_RDTSCP: return vmxHCExitRdtscpNested(pVCpu, pVmxTransient);
5492	case VMX_EXIT_RDMSR: return vmxHCExitRdmsrNested(pVCpu, pVmxTransient);
5493	case VMX_EXIT_WRMSR: return vmxHCExitWrmsrNested(pVCpu, pVmxTransient);
5494	case VMX_EXIT_INVLPG: return vmxHCExitInvlpgNested(pVCpu, pVmxTransient);
5495	case VMX_EXIT_INVPCID: return vmxHCExitInvpcidNested(pVCpu, pVmxTransient);
5496	case VMX_EXIT_TASK_SWITCH: return vmxHCExitTaskSwitchNested(pVCpu, pVmxTransient);
5497	case VMX_EXIT_WBINVD: return vmxHCExitWbinvdNested(pVCpu, pVmxTransient);
5498	case VMX_EXIT_MTF: return vmxHCExitMtfNested(pVCpu, pVmxTransient);
5499	case VMX_EXIT_APIC_ACCESS: return vmxHCExitApicAccessNested(pVCpu, pVmxTransient);
5500	case VMX_EXIT_APIC_WRITE: return vmxHCExitApicWriteNested(pVCpu, pVmxTransient);
5501	case VMX_EXIT_VIRTUALIZED_EOI: return vmxHCExitVirtEoiNested(pVCpu, pVmxTransient);
5502	case VMX_EXIT_MOV_CRX: return vmxHCExitMovCRxNested(pVCpu, pVmxTransient);
5503	case VMX_EXIT_INT_WINDOW: return vmxHCExitIntWindowNested(pVCpu, pVmxTransient);
5504	case VMX_EXIT_NMI_WINDOW: return vmxHCExitNmiWindowNested(pVCpu, pVmxTransient);
5505	case VMX_EXIT_TPR_BELOW_THRESHOLD: return vmxHCExitTprBelowThresholdNested(pVCpu, pVmxTransient);
5506	case VMX_EXIT_MWAIT: return vmxHCExitMwaitNested(pVCpu, pVmxTransient);
5507	case VMX_EXIT_MONITOR: return vmxHCExitMonitorNested(pVCpu, pVmxTransient);
5508	case VMX_EXIT_PAUSE: return vmxHCExitPauseNested(pVCpu, pVmxTransient);
5509
5510	case VMX_EXIT_PREEMPT_TIMER:
5511	{
5512	/** @todo NSTVMX: Preempt timer. */
5513	return vmxHCExitPreemptTimer(pVCpu, pVmxTransient);
5514	}
5515
5516	case VMX_EXIT_MOV_DRX: return vmxHCExitMovDRxNested(pVCpu, pVmxTransient);
5517	case VMX_EXIT_RDPMC: return vmxHCExitRdpmcNested(pVCpu, pVmxTransient);
5518
5519	case VMX_EXIT_VMREAD:
5520	case VMX_EXIT_VMWRITE: return vmxHCExitVmreadVmwriteNested(pVCpu, pVmxTransient);
5521
5522	case VMX_EXIT_TRIPLE_FAULT: return vmxHCExitTripleFaultNested(pVCpu, pVmxTransient);
5523	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return vmxHCExitErrInvalidGuestStateNested(pVCpu, pVmxTransient);
5524
5525	case VMX_EXIT_INIT_SIGNAL:
5526	case VMX_EXIT_SIPI:
5527	case VMX_EXIT_IO_SMI:
5528	case VMX_EXIT_SMI:
5529	case VMX_EXIT_ERR_MSR_LOAD:
5530	case VMX_EXIT_ERR_MACHINE_CHECK:
5531	case VMX_EXIT_PML_FULL:
5532	case VMX_EXIT_RSM:
5533	default:
5534	return vmxHCExitErrUnexpected(pVCpu, pVmxTransient);
5535	}
5536	}
5537	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
5538
5539
5540	/** @name VM-exit helpers.
5541	* @{
5542	*/
5543	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
5544	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit helpers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
5545	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
5546
5547	/** Macro for VM-exits called unexpectedly. */
5548	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
5549	do { \
5550	VCPU_2_VMXSTATE((a_pVCpu)).u32HMError = (a_HmError); \
5551	return VERR_VMX_UNEXPECTED_EXIT; \
5552	} while (0)
5553
5554	#ifdef VBOX_STRICT
5555	# ifndef IN_NEM_DARWIN
5556	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
5557	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
5558	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
5559
5560	# define HMVMX_ASSERT_PREEMPT_CPUID() \
5561	do { \
5562	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
5563	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
5564	} while (0)
5565
5566	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
5567	do { \
5568	AssertPtr((a_pVCpu)); \
5569	AssertPtr((a_pVmxTransient)); \
5570	Assert( (a_pVmxTransient)->fVMEntryFailed == false \
5571	\|\| (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE \
5572	\|\| (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_MSR_LOAD \
5573	\|\| (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_MACHINE_CHECK); \
5574	Assert((a_pVmxTransient)->pVmcsInfo); \
5575	Assert(ASMIntAreEnabled()); \
5576	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
5577	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
5578	Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
5579	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
5580	if (!VMMRZCallRing3IsEnabled((a_pVCpu))) \
5581	HMVMX_ASSERT_PREEMPT_CPUID(); \
5582	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
5583	} while (0)
5584	# else
5585	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() do { } while(0)
5586	# define HMVMX_ASSERT_PREEMPT_CPUID() do { } while(0)
5587	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
5588	do { \
5589	AssertPtr((a_pVCpu)); \
5590	AssertPtr((a_pVmxTransient)); \
5591	Assert( (a_pVmxTransient)->fVMEntryFailed == false \
5592	\|\| (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE \
5593	\|\| (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_MSR_LOAD \
5594	\|\| (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_MACHINE_CHECK); \
5595	Assert((a_pVmxTransient)->pVmcsInfo); \
5596	Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
5597	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
5598	} while (0)
5599	# endif
5600
5601	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
5602	do { \
5603	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
5604	Assert((a_pVmxTransient)->fIsNestedGuest); \
5605	} while (0)
5606
5607	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
5608	do { \
5609	Log4Func(("\n")); \
5610	} while (0)
5611	#else
5612	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
5613	do { \
5614	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
5615	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
5616	} while (0)
5617
5618	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
5619	do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
5620
5621	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
5622	#endif
5623
5624	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5625	/** Macro that does the necessary privilege checks and intercepted VM-exits for
5626	* guests that attempted to execute a VMX instruction. */
5627	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
5628	do \
5629	{ \
5630	VBOXSTRICTRC rcStrictTmp = vmxHCCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
5631	if (rcStrictTmp == VINF_SUCCESS) \
5632	{ /* likely */ } \
5633	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
5634	{ \
5635	Assert((a_pVCpu)->hm.s.Event.fPending); \
5636	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
5637	return VINF_SUCCESS; \
5638	} \
5639	else \
5640	{ \
5641	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
5642	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
5643	} \
5644	} while (0)
5645
5646	/** Macro that decodes a memory operand for an VM-exit caused by an instruction. */
5647	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
5648	do \
5649	{ \
5650	VBOXSTRICTRC rcStrictTmp = vmxHCDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
5651	(a_pGCPtrEffAddr)); \
5652	if (rcStrictTmp == VINF_SUCCESS) \
5653	{ /* likely */ } \
5654	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
5655	{ \
5656	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
5657	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
5658	NOREF(uXcptTmp); \
5659	return VINF_SUCCESS; \
5660	} \
5661	else \
5662	{ \
5663	Log4Func(("vmxHCDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
5664	return rcStrictTmp; \
5665	} \
5666	} while (0)
5667	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
5668
5669
5670	/**
5671	* Advances the guest RIP by the specified number of bytes.
5672	*
5673	* @param pVCpu The cross context virtual CPU structure.
5674	* @param cbInstr Number of bytes to advance the RIP by.
5675	*
5676	* @remarks No-long-jump zone!!!
5677	*/
5678	DECLINLINE(void) vmxHCAdvanceGuestRipBy(PVMCPUCC pVCpu, uint32_t cbInstr)
5679	{
5680	/* Advance the RIP. */
5681	pVCpu->cpum.GstCtx.rip += cbInstr;
5682	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP);
5683
5684	/* Update interrupt inhibition. */
5685	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
5686	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
5687	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5688	}
5689
5690
5691	/**
5692	* Advances the guest RIP after reading it from the VMCS.
5693	*
5694	* @returns VBox status code, no informational status codes.
5695	* @param pVCpu The cross context virtual CPU structure.
5696	* @param pVmxTransient The VMX-transient structure.
5697	*
5698	* @remarks No-long-jump zone!!!
5699	*/
5700	static int vmxHCAdvanceGuestRip(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5701	{
5702	vmxHCReadToTransientSlow<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
5703	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
5704	AssertRCReturn(rc, rc);
5705
5706	vmxHCAdvanceGuestRipBy(pVCpu, pVmxTransient->cbExitInstr);
5707	return VINF_SUCCESS;
5708	}
5709
5710
5711	/**
5712	* Handle a condition that occurred while delivering an event through the guest or
5713	* nested-guest IDT.
5714	*
5715	* @returns Strict VBox status code (i.e. informational status codes too).
5716	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5717	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5718	* to continue execution of the guest which will delivery the \#DF.
5719	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5720	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5721	*
5722	* @param pVCpu The cross context virtual CPU structure.
5723	* @param pVmxTransient The VMX-transient structure.
5724	*
5725	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
5726	* Additionally, HMVMX_READ_EXIT_QUALIFICATION is required if the VM-exit
5727	* is due to an EPT violation, PML full or SPP-related event.
5728	*
5729	* @remarks No-long-jump zone!!!
5730	*/
5731	static VBOXSTRICTRC vmxHCCheckExitDueToEventDelivery(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5732	{
5733	Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending);
5734	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
5735	if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
5736	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
5737	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
5738	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_EXIT_QUALIFICATION);
5739
5740	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5741	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5742	uint32_t const uIdtVectorInfo = pVmxTransient->uIdtVectoringInfo;
5743	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
5744	if (VMX_IDT_VECTORING_INFO_IS_VALID(uIdtVectorInfo))
5745	{
5746	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(uIdtVectorInfo);
5747	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(uIdtVectorInfo);
5748
5749	/*
5750	* If the event was a software interrupt (generated with INT n) or a software exception
5751	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
5752	* can handle the VM-exit and continue guest execution which will re-execute the
5753	* instruction rather than re-injecting the exception, as that can cause premature
5754	* trips to ring-3 before injection and involve TRPM which currently has no way of
5755	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
5756	* the problem).
5757	*/
5758	IEMXCPTRAISE enmRaise;
5759	IEMXCPTRAISEINFO fRaiseInfo;
5760	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5761	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5762	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5763	{
5764	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5765	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5766	}
5767	else if (VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo))
5768	{
5769	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
5770	uint8_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
5771	Assert(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT);
5772
5773	uint32_t const fIdtVectorFlags = vmxHCGetIemXcptFlags(uIdtVector, uIdtVectorType);
5774	uint32_t const fExitVectorFlags = vmxHCGetIemXcptFlags(uExitVector, uExitVectorType);
5775
5776	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5777
5778	/* Determine a vectoring #PF condition, see comment in vmxHCExitXcptPF(). */
5779	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
5780	{
5781	pVmxTransient->fVectoringPF = true;
5782	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5783	}
5784	}
5785	else
5786	{
5787	/*
5788	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5789	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5790	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
5791	*/
5792	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5793	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5794	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5795	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5796	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5797	}
5798
5799	/*
5800	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5801	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5802	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5803	* subsequent VM-entry would fail, see @bugref{7445}.
5804	*
5805	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception".
5806	*/
5807	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5808	&& enmRaise == IEMXCPTRAISE_PREV_EVENT
5809	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
5810	&& CPUMIsGuestNmiBlocking(pVCpu))
5811	{
5812	CPUMSetGuestNmiBlocking(pVCpu, false);
5813	}
5814
5815	switch (enmRaise)
5816	{
5817	case IEMXCPTRAISE_CURRENT_XCPT:
5818	{
5819	Log4Func(("IDT: Pending secondary Xcpt: idtinfo=%#RX64 exitinfo=%#RX64\n", uIdtVectorInfo, uExitIntInfo));
5820	Assert(rcStrict == VINF_SUCCESS);
5821	break;
5822	}
5823
5824	case IEMXCPTRAISE_PREV_EVENT:
5825	{
5826	uint32_t u32ErrCode;
5827	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(uIdtVectorInfo))
5828	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5829	else
5830	u32ErrCode = 0;
5831
5832	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see vmxHCExitXcptPF(). */
5833	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectReflect);
5834	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(uIdtVectorInfo), 0 /* cbInstr */, u32ErrCode,
5835	pVCpu->cpum.GstCtx.cr2);
5836
5837	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
5838	VCPU_2_VMXSTATE(pVCpu).Event.u32ErrCode));
5839	Assert(rcStrict == VINF_SUCCESS);
5840	break;
5841	}
5842
5843	case IEMXCPTRAISE_REEXEC_INSTR:
5844	Assert(rcStrict == VINF_SUCCESS);
5845	break;
5846
5847	case IEMXCPTRAISE_DOUBLE_FAULT:
5848	{
5849	/*
5850	* Determine a vectoring double #PF condition. Used later, when PGM evaluates the
5851	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
5852	*/
5853	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5854	{
5855	pVmxTransient->fVectoringDoublePF = true;
5856	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
5857	pVCpu->cpum.GstCtx.cr2));
5858	rcStrict = VINF_SUCCESS;
5859	}
5860	else
5861	{
5862	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectConvertDF);
5863	vmxHCSetPendingXcptDF(pVCpu);
5864	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
5865	uIdtVector, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
5866	rcStrict = VINF_HM_DOUBLE_FAULT;
5867	}
5868	break;
5869	}
5870
5871	case IEMXCPTRAISE_TRIPLE_FAULT:
5872	{
5873	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector,
5874	VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
5875	rcStrict = VINF_EM_RESET;
5876	break;
5877	}
5878
5879	case IEMXCPTRAISE_CPU_HANG:
5880	{
5881	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
5882	rcStrict = VERR_EM_GUEST_CPU_HANG;
5883	break;
5884	}
5885
5886	default:
5887	{
5888	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
5889	rcStrict = VERR_VMX_IPE_2;
5890	break;
5891	}
5892	}
5893	}
5894	else if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
5895	&& !CPUMIsGuestNmiBlocking(pVCpu))
5896	{
5897	if ( VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo)
5898	&& VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo) != X86_XCPT_DF
5899	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(uExitIntInfo))
5900	{
5901	/*
5902	* Execution of IRET caused a fault when NMI blocking was in effect (i.e we're in
5903	* the guest or nested-guest NMI handler). We need to set the block-by-NMI field so
5904	* that virtual NMIs remain blocked until the IRET execution is completed.
5905	*
5906	* See Intel spec. 31.7.1.2 "Resuming Guest Software After Handling An Exception".
5907	*/
5908	CPUMSetGuestNmiBlocking(pVCpu, true);
5909	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
5910	}
5911	else if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
5912	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
5913	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
5914	{
5915	/*
5916	* Execution of IRET caused an EPT violation, page-modification log-full event or
5917	* SPP-related event VM-exit when NMI blocking was in effect (i.e. we're in the
5918	* guest or nested-guest NMI handler). We need to set the block-by-NMI field so
5919	* that virtual NMIs remain blocked until the IRET execution is completed.
5920	*
5921	* See Intel spec. 27.2.3 "Information about NMI unblocking due to IRET"
5922	*/
5923	if (VMX_EXIT_QUAL_EPT_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitQual))
5924	{
5925	CPUMSetGuestNmiBlocking(pVCpu, true);
5926	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
5927	}
5928	}
5929	}
5930
5931	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
5932	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
5933	return rcStrict;
5934	}
5935
5936
5937	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5938	/**
5939	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
5940	* guest attempting to execute a VMX instruction.
5941	*
5942	* @returns Strict VBox status code (i.e. informational status codes too).
5943	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5944	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
5945	*
5946	* @param pVCpu The cross context virtual CPU structure.
5947	* @param uExitReason The VM-exit reason.
5948	*
5949	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
5950	* @remarks No-long-jump zone!!!
5951	*/
5952	static VBOXSTRICTRC vmxHCCheckExitDueToVmxInstr(PVMCPUCC pVCpu, uint32_t uExitReason)
5953	{
5954	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
5955	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
5956
5957	/*
5958	* The physical CPU would have already checked the CPU mode/code segment.
5959	* We shall just assert here for paranoia.
5960	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
5961	*/
5962	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
5963	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5964	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
5965
5966	if (uExitReason == VMX_EXIT_VMXON)
5967	{
5968	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5969
5970	/*
5971	* We check CR4.VMXE because it is required to be always set while in VMX operation
5972	* by physical CPUs and our CR4 read-shadow is only consulted when executing specific
5973	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
5974	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
5975	*/
5976	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
5977	{
5978	Log4Func(("CR4.VMXE is not set -> #UD\n"));
5979	vmxHCSetPendingXcptUD(pVCpu);
5980	return VINF_HM_PENDING_XCPT;
5981	}
5982	}
5983	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
5984	{
5985	/*
5986	* The guest has not entered VMX operation but attempted to execute a VMX instruction
5987	* (other than VMXON), we need to raise a #UD.
5988	*/
5989	Log4Func(("Not in VMX root mode -> #UD\n"));
5990	vmxHCSetPendingXcptUD(pVCpu);
5991	return VINF_HM_PENDING_XCPT;
5992	}
5993
5994	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
5995	return VINF_SUCCESS;
5996	}
5997
5998
5999	/**
6000	* Decodes the memory operand of an instruction that caused a VM-exit.
6001	*
6002	* The Exit qualification field provides the displacement field for memory
6003	* operand instructions, if any.
6004	*
6005	* @returns Strict VBox status code (i.e. informational status codes too).
6006	* @retval VINF_SUCCESS if the operand was successfully decoded.
6007	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
6008	* operand.
6009	* @param pVCpu The cross context virtual CPU structure.
6010	* @param uExitInstrInfo The VM-exit instruction information field.
6011	* @param enmMemAccess The memory operand's access type (read or write).
6012	* @param GCPtrDisp The instruction displacement field, if any. For
6013	* RIP-relative addressing pass RIP + displacement here.
6014	* @param pGCPtrMem Where to store the effective destination memory address.
6015	*
6016	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
6017	* virtual-8086 mode hence skips those checks while verifying if the
6018	* segment is valid.
6019	*/
6020	static VBOXSTRICTRC vmxHCDecodeMemOperand(PVMCPUCC pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
6021	PRTGCPTR pGCPtrMem)
6022	{
6023	Assert(pGCPtrMem);
6024	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
6025	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
6026	\| CPUMCTX_EXTRN_CR0);
6027
6028	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
6029	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
6030	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
6031
6032	VMXEXITINSTRINFO ExitInstrInfo;
6033	ExitInstrInfo.u = uExitInstrInfo;
6034	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
6035	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
6036	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
6037	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
6038	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
6039	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
6040	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
6041	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
6042	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
6043
6044	/*
6045	* Validate instruction information.
6046	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
6047	*/
6048	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
6049	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
6050	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
6051	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
6052	AssertLogRelMsgReturn(fIsMemOperand,
6053	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
6054
6055	/*
6056	* Compute the complete effective address.
6057	*
6058	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
6059	* See AMD spec. 4.5.2 "Segment Registers".
6060	*/
6061	RTGCPTR GCPtrMem = GCPtrDisp;
6062	if (fBaseRegValid)
6063	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
6064	if (fIdxRegValid)
6065	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
6066
6067	RTGCPTR const GCPtrOff = GCPtrMem;
6068	if ( !fIsLongMode
6069	\|\| iSegReg >= X86_SREG_FS)
6070	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
6071	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
6072
6073	/*
6074	* Validate effective address.
6075	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
6076	*/
6077	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
6078	Assert(cbAccess > 0);
6079	if (fIsLongMode)
6080	{
6081	if (X86_IS_CANONICAL(GCPtrMem))
6082	{
6083	*pGCPtrMem = GCPtrMem;
6084	return VINF_SUCCESS;
6085	}
6086
6087	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
6088	* "Data Limit Checks in 64-bit Mode". */
6089	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
6090	vmxHCSetPendingXcptGP(pVCpu, 0);
6091	return VINF_HM_PENDING_XCPT;
6092	}
6093
6094	/*
6095	* This is a watered down version of iemMemApplySegment().
6096	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
6097	* and segment CPL/DPL checks are skipped.
6098	*/
6099	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
6100	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
6101	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
6102
6103	/* Check if the segment is present and usable. */
6104	if ( pSel->Attr.n.u1Present
6105	&& !pSel->Attr.n.u1Unusable)
6106	{
6107	Assert(pSel->Attr.n.u1DescType);
6108	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
6109	{
6110	/* Check permissions for the data segment. */
6111	if ( enmMemAccess == VMXMEMACCESS_WRITE
6112	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
6113	{
6114	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6115	vmxHCSetPendingXcptGP(pVCpu, iSegReg);
6116	return VINF_HM_PENDING_XCPT;
6117	}
6118
6119	/* Check limits if it's a normal data segment. */
6120	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
6121	{
6122	if ( GCPtrFirst32 > pSel->u32Limit
6123	\|\| GCPtrLast32 > pSel->u32Limit)
6124	{
6125	Log4Func(("Data segment limit exceeded. "
6126	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6127	GCPtrLast32, pSel->u32Limit));
6128	if (iSegReg == X86_SREG_SS)
6129	vmxHCSetPendingXcptSS(pVCpu, 0);
6130	else
6131	vmxHCSetPendingXcptGP(pVCpu, 0);
6132	return VINF_HM_PENDING_XCPT;
6133	}
6134	}
6135	else
6136	{
6137	/* Check limits if it's an expand-down data segment.
6138	Note! The upper boundary is defined by the B bit, not the G bit! */
6139	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
6140	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
6141	{
6142	Log4Func(("Expand-down data segment limit exceeded. "
6143	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6144	GCPtrLast32, pSel->u32Limit));
6145	if (iSegReg == X86_SREG_SS)
6146	vmxHCSetPendingXcptSS(pVCpu, 0);
6147	else
6148	vmxHCSetPendingXcptGP(pVCpu, 0);
6149	return VINF_HM_PENDING_XCPT;
6150	}
6151	}
6152	}
6153	else
6154	{
6155	/* Check permissions for the code segment. */
6156	if ( enmMemAccess == VMXMEMACCESS_WRITE
6157	\|\| ( enmMemAccess == VMXMEMACCESS_READ
6158	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
6159	{
6160	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6161	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6162	vmxHCSetPendingXcptGP(pVCpu, 0);
6163	return VINF_HM_PENDING_XCPT;
6164	}
6165
6166	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6167	if ( GCPtrFirst32 > pSel->u32Limit
6168	\|\| GCPtrLast32 > pSel->u32Limit)
6169	{
6170	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6171	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6172	if (iSegReg == X86_SREG_SS)
6173	vmxHCSetPendingXcptSS(pVCpu, 0);
6174	else
6175	vmxHCSetPendingXcptGP(pVCpu, 0);
6176	return VINF_HM_PENDING_XCPT;
6177	}
6178	}
6179	}
6180	else
6181	{
6182	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6183	vmxHCSetPendingXcptGP(pVCpu, 0);
6184	return VINF_HM_PENDING_XCPT;
6185	}
6186
6187	*pGCPtrMem = GCPtrMem;
6188	return VINF_SUCCESS;
6189	}
6190	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
6191
6192
6193	/**
6194	* VM-exit helper for LMSW.
6195	*/
6196	static VBOXSTRICTRC vmxHCExitLmsw(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst)
6197	{
6198	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
6199	AssertRCReturn(rc, rc);
6200
6201	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
6202	AssertMsg( rcStrict == VINF_SUCCESS
6203	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6204
6205	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
6206	if (rcStrict == VINF_IEM_RAISED_XCPT)
6207	{
6208	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6209	rcStrict = VINF_SUCCESS;
6210	}
6211
6212	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitLmsw);
6213	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6214	return rcStrict;
6215	}
6216
6217
6218	/**
6219	* VM-exit helper for CLTS.
6220	*/
6221	static VBOXSTRICTRC vmxHCExitClts(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
6222	{
6223	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
6224	AssertRCReturn(rc, rc);
6225
6226	VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
6227	AssertMsg( rcStrict == VINF_SUCCESS
6228	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6229
6230	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
6231	if (rcStrict == VINF_IEM_RAISED_XCPT)
6232	{
6233	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6234	rcStrict = VINF_SUCCESS;
6235	}
6236
6237	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitClts);
6238	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6239	return rcStrict;
6240	}
6241
6242
6243	/**
6244	* VM-exit helper for MOV from CRx (CRx read).
6245	*/
6246	static VBOXSTRICTRC vmxHCExitMovFromCrX(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
6247	{
6248	Assert(iCrReg < 16);
6249	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
6250
6251	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
6252	AssertRCReturn(rc, rc);
6253
6254	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
6255	AssertMsg( rcStrict == VINF_SUCCESS
6256	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6257
6258	if (iGReg == X86_GREG_xSP)
6259	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
6260	else
6261	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
6262	#ifdef VBOX_WITH_STATISTICS
6263	switch (iCrReg)
6264	{
6265	case 0: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR0Read); break;
6266	case 2: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR2Read); break;
6267	case 3: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR3Read); break;
6268	case 4: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR4Read); break;
6269	case 8: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR8Read); break;
6270	}
6271	#endif
6272	Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
6273	return rcStrict;
6274	}
6275
6276
6277	/**
6278	* VM-exit helper for MOV to CRx (CRx write).
6279	*/
6280	static VBOXSTRICTRC vmxHCExitMovToCrX(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
6281	{
6282	HMVMX_CPUMCTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6283
6284	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
6285	AssertMsg( rcStrict == VINF_SUCCESS
6286	\|\| rcStrict == VINF_IEM_RAISED_XCPT
6287	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6288
6289	switch (iCrReg)
6290	{
6291	case 0:
6292	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0
6293	\| HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
6294	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR0Write);
6295	Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
6296	break;
6297
6298	case 2:
6299	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR2Write);
6300	/* Nothing to do here, CR2 it's not part of the VMCS. */
6301	break;
6302
6303	case 3:
6304	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
6305	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR3Write);
6306	Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
6307	break;
6308
6309	case 4:
6310	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
6311	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR4Write);
6312	#ifndef IN_NEM_DARWIN
6313	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
6314	pVCpu->cpum.GstCtx.cr4, pVCpu->hmr0.s.fLoadSaveGuestXcr0));
6315	#else
6316	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr4));
6317	#endif
6318	break;
6319
6320	case 8:
6321	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged,
6322	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
6323	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR8Write);
6324	break;
6325
6326	default:
6327	AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
6328	break;
6329	}
6330
6331	if (rcStrict == VINF_IEM_RAISED_XCPT)
6332	{
6333	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6334	rcStrict = VINF_SUCCESS;
6335	}
6336	return rcStrict;
6337	}
6338
6339
6340	/**
6341	* VM-exit exception handler for \#PF (Page-fault exception).
6342	*
6343	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
6344	*/
6345	static VBOXSTRICTRC vmxHCExitXcptPF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6346	{
6347	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
6348	vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
6349
6350	#ifndef IN_NEM_DARWIN
6351	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6352	if (!VM_IS_VMX_NESTED_PAGING(pVM))
6353	{ /* likely */ }
6354	else
6355	#endif
6356	{
6357	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF) && !defined(IN_NEM_DARWIN)
6358	Assert(pVmxTransient->fIsNestedGuest \|\| pVCpu->hmr0.s.fUsingDebugLoop);
6359	#endif
6360	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
6361	if (!pVmxTransient->fVectoringDoublePF)
6362	{
6363	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
6364	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
6365	}
6366	else
6367	{
6368	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
6369	Assert(!pVmxTransient->fIsNestedGuest);
6370	vmxHCSetPendingXcptDF(pVCpu);
6371	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
6372	}
6373	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestPF);
6374	return VINF_SUCCESS;
6375	}
6376
6377	Assert(!pVmxTransient->fIsNestedGuest);
6378
6379	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
6380	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
6381	if (pVmxTransient->fVectoringPF)
6382	{
6383	Assert(VCPU_2_VMXSTATE(pVCpu).Event.fPending);
6384	return VINF_EM_RAW_INJECT_TRPM_EVENT;
6385	}
6386
6387	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6388	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
6389	AssertRCReturn(rc, rc);
6390
6391	Log4Func(("#PF: cs:rip=%#04x:%08RX64 err_code=%#RX32 exit_qual=%#RX64 cr3=%#RX64\n", pCtx->cs.Sel, pCtx->rip,
6392	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual, pCtx->cr3));
6393
6394	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
6395	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
6396
6397	Log4Func(("#PF: rc=%Rrc\n", rc));
6398	if (rc == VINF_SUCCESS)
6399	{
6400	/*
6401	* This is typically a shadow page table sync or a MMIO instruction. But we may have
6402	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
6403	*/
6404	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
6405	TRPMResetTrap(pVCpu);
6406	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitShadowPF);
6407	return rc;
6408	}
6409
6410	if (rc == VINF_EM_RAW_GUEST_TRAP)
6411	{
6412	if (!pVmxTransient->fVectoringDoublePF)
6413	{
6414	/* It's a guest page fault and needs to be reflected to the guest. */
6415	uint32_t const uGstErrorCode = TRPMGetErrorCode(pVCpu);
6416	TRPMResetTrap(pVCpu);
6417	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false; /* In case it's a contributory #PF. */
6418	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
6419	uGstErrorCode, pVmxTransient->uExitQual);
6420	}
6421	else
6422	{
6423	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
6424	TRPMResetTrap(pVCpu);
6425	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
6426	vmxHCSetPendingXcptDF(pVCpu);
6427	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
6428	}
6429
6430	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestPF);
6431	return VINF_SUCCESS;
6432	}
6433
6434	TRPMResetTrap(pVCpu);
6435	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitShadowPFEM);
6436	return rc;
6437	}
6438
6439
6440	/**
6441	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
6442	*
6443	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
6444	*/
6445	static VBOXSTRICTRC vmxHCExitXcptMF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6446	{
6447	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
6448	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestMF);
6449
6450	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
6451	AssertRCReturn(rc, rc);
6452
6453	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
6454	{
6455	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
6456	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
6457
6458	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
6459	* provides VM-exit instruction length. If this causes problem later,
6460	* disassemble the instruction like it's done on AMD-V. */
6461	int rc2 = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
6462	AssertRCReturn(rc2, rc2);
6463	return rc;
6464	}
6465
6466	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbExitInstr,
6467	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
6468	return VINF_SUCCESS;
6469	}
6470
6471
6472	/**
6473	* VM-exit exception handler for \#BP (Breakpoint exception).
6474	*
6475	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
6476	*/
6477	static VBOXSTRICTRC vmxHCExitXcptBP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6478	{
6479	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
6480	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestBP);
6481
6482	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
6483	AssertRCReturn(rc, rc);
6484
6485	VBOXSTRICTRC rcStrict;
6486	if (!pVmxTransient->fIsNestedGuest)
6487	rcStrict = DBGFTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx));
6488	else
6489	rcStrict = VINF_EM_RAW_GUEST_TRAP;
6490
6491	if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
6492	{
6493	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
6494	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
6495	rcStrict = VINF_SUCCESS;
6496	}
6497
6498	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_DBG_BREAKPOINT);
6499	return rcStrict;
6500	}
6501
6502
6503	/**
6504	* VM-exit exception handler for \#AC (Alignment-check exception).
6505	*
6506	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
6507	*/
6508	static VBOXSTRICTRC vmxHCExitXcptAC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6509	{
6510	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
6511
6512	/*
6513	* Detect #ACs caused by host having enabled split-lock detection.
6514	* Emulate such instructions.
6515	*/
6516	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo,
6517	CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS);
6518	AssertRCReturn(rc, rc);
6519	/** @todo detect split lock in cpu feature? */
6520	if ( /* 1. If 486-style alignment checks aren't enabled, then this must be a split-lock exception */
6521	!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM)
6522	/* 2. #AC cannot happen in rings 0-2 except for split-lock detection. */
6523	\|\| CPUMGetGuestCPL(pVCpu) != 3
6524	/* 3. When the EFLAGS.AC != 0 this can only be a split-lock case. */
6525	\|\| !(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_AC) )
6526	{
6527	/*
6528	* Check for debug/trace events and import state accordingly.
6529	*/
6530	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestACSplitLock);
6531	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6532	if ( !DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_VMX_SPLIT_LOCK)
6533	#ifndef IN_NEM_DARWIN
6534	&& !VBOXVMM_VMX_SPLIT_LOCK_ENABLED()
6535	#endif
6536	)
6537	{
6538	if (pVM->cCpus == 1)
6539	{
6540	#if 0 /** @todo r=bird: This is potentially wrong. Might have to just do a whole state sync above and mark everything changed to be safe... */
6541	rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
6542	#else
6543	rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
6544	#endif
6545	AssertRCReturn(rc, rc);
6546	}
6547	}
6548	else
6549	{
6550	rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
6551	AssertRCReturn(rc, rc);
6552
6553	VBOXVMM_XCPT_DF(pVCpu, &pVCpu->cpum.GstCtx);
6554
6555	if (DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_VMX_SPLIT_LOCK))
6556	{
6557	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, DBGFEVENT_VMX_SPLIT_LOCK, DBGFEVENTCTX_HM, 0);
6558	if (rcStrict != VINF_SUCCESS)
6559	return rcStrict;
6560	}
6561	}
6562
6563	/*
6564	* Emulate the instruction.
6565	*
6566	* We have to ignore the LOCK prefix here as we must not retrigger the
6567	* detection on the host. This isn't all that satisfactory, though...
6568	*/
6569	if (pVM->cCpus == 1)
6570	{
6571	Log8Func(("cs:rip=%#04x:%08RX64 rflags=%#RX64 cr0=%#RX64 split-lock #AC\n", pVCpu->cpum.GstCtx.cs.Sel,
6572	pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags, pVCpu->cpum.GstCtx.cr0));
6573
6574	/** @todo For SMP configs we should do a rendezvous here. */
6575	VBOXSTRICTRC rcStrict = IEMExecOneIgnoreLock(pVCpu);
6576	if (rcStrict == VINF_SUCCESS)
6577	#if 0 /** @todo r=bird: This is potentially wrong. Might have to just do a whole state sync above and mark everything changed to be safe... */
6578	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged,
6579	HM_CHANGED_GUEST_RIP
6580	\| HM_CHANGED_GUEST_RFLAGS
6581	\| HM_CHANGED_GUEST_GPRS_MASK
6582	\| HM_CHANGED_GUEST_CS
6583	\| HM_CHANGED_GUEST_SS);
6584	#else
6585	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
6586	#endif
6587	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6588	{
6589	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6590	rcStrict = VINF_SUCCESS;
6591	}
6592	return rcStrict;
6593	}
6594	Log8Func(("cs:rip=%#04x:%08RX64 rflags=%#RX64 cr0=%#RX64 split-lock #AC -> VINF_EM_EMULATE_SPLIT_LOCK\n",
6595	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags, pVCpu->cpum.GstCtx.cr0));
6596	return VINF_EM_EMULATE_SPLIT_LOCK;
6597	}
6598
6599	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestAC);
6600	Log8Func(("cs:rip=%#04x:%08RX64 rflags=%#RX64 cr0=%#RX64 cpl=%d -> #AC\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
6601	pVCpu->cpum.GstCtx.rflags, pVCpu->cpum.GstCtx.cr0, CPUMGetGuestCPL(pVCpu) ));
6602
6603	/* Re-inject it. We'll detect any nesting before getting here. */
6604	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
6605	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
6606	return VINF_SUCCESS;
6607	}
6608
6609
6610	/**
6611	* VM-exit exception handler for \#DB (Debug exception).
6612	*
6613	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
6614	*/
6615	static VBOXSTRICTRC vmxHCExitXcptDB(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6616	{
6617	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
6618	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDB);
6619
6620	/*
6621	* Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
6622	*/
6623	vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
6624
6625	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
6626	uint64_t const uDR6 = X86_DR6_INIT_VAL
6627	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
6628	\| X86_DR6_BD \| X86_DR6_BS));
6629
6630	int rc;
6631	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6632	if (!pVmxTransient->fIsNestedGuest)
6633	{
6634	rc = DBGFTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, VCPU_2_VMXSTATE(pVCpu).fSingleInstruction);
6635
6636	/*
6637	* Prevents stepping twice over the same instruction when the guest is stepping using
6638	* EFLAGS.TF and the hypervisor debugger is stepping using MTF.
6639	* Testcase: DOSQEMM, break (using "ba x 1") at cs:rip 0x70:0x774 and step (using "t").
6640	*/
6641	if ( rc == VINF_EM_DBG_STEPPED
6642	&& (pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG))
6643	{
6644	Assert(VCPU_2_VMXSTATE(pVCpu).fSingleInstruction);
6645	rc = VINF_EM_RAW_GUEST_TRAP;
6646	}
6647	}
6648	else
6649	rc = VINF_EM_RAW_GUEST_TRAP;
6650	Log6Func(("rc=%Rrc\n", rc));
6651	if (rc == VINF_EM_RAW_GUEST_TRAP)
6652	{
6653	/*
6654	* The exception was for the guest. Update DR6, DR7.GD and
6655	* IA32_DEBUGCTL.LBR before forwarding it.
6656	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
6657	*/
6658	#ifndef IN_NEM_DARWIN
6659	VMMRZCallRing3Disable(pVCpu);
6660	HM_DISABLE_PREEMPT(pVCpu);
6661
6662	pCtx->dr[6] &= ~X86_DR6_B_MASK;
6663	pCtx->dr[6] \|= uDR6;
6664	if (CPUMIsGuestDebugStateActive(pVCpu))
6665	ASMSetDR6(pCtx->dr[6]);
6666
6667	HM_RESTORE_PREEMPT();
6668	VMMRZCallRing3Enable(pVCpu);
6669	#else
6670	/** @todo */
6671	#endif
6672
6673	rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
6674	AssertRCReturn(rc, rc);
6675
6676	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
6677	pCtx->dr[7] &= ~(uint64_t)X86_DR7_GD;
6678
6679	/* Paranoia. */
6680	pCtx->dr[7] &= ~(uint64_t)X86_DR7_RAZ_MASK;
6681	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
6682
6683	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
6684	AssertRC(rc);
6685
6686	/*
6687	* Raise #DB in the guest.
6688	*
6689	* It is important to reflect exactly what the VM-exit gave us (preserving the
6690	* interruption-type) rather than use vmxHCSetPendingXcptDB() as the #DB could've
6691	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
6692	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
6693	*
6694	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
6695	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6696	*/
6697	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
6698	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
6699	return VINF_SUCCESS;
6700	}
6701
6702	/*
6703	* Not a guest trap, must be a hypervisor related debug event then.
6704	* Update DR6 in case someone is interested in it.
6705	*/
6706	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
6707	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
6708	CPUMSetHyperDR6(pVCpu, uDR6);
6709
6710	return rc;
6711	}
6712
6713
6714	/**
6715	* Hacks its way around the lovely mesa driver's backdoor accesses.
6716	*
6717	* @sa hmR0SvmHandleMesaDrvGp.
6718	*/
6719	static int vmxHCHandleMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
6720	{
6721	LogFunc(("cs:rip=%#04x:%08RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
6722	RT_NOREF(pCtx);
6723
6724	/* For now we'll just skip the instruction. */
6725	return vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
6726	}
6727
6728
6729	/**
6730	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
6731	* backdoor logging w/o checking what it is running inside.
6732	*
6733	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
6734	* backdoor port and magic numbers loaded in registers.
6735	*
6736	* @returns true if it is, false if it isn't.
6737	* @sa hmR0SvmIsMesaDrvGp.
6738	*/
6739	DECLINLINE(bool) vmxHCIsMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
6740	{
6741	/* 0xed: IN eAX,dx */
6742	uint8_t abInstr[1];
6743	if (pVmxTransient->cbExitInstr != sizeof(abInstr))
6744	return false;
6745
6746	/* Check that it is #GP(0). */
6747	if (pVmxTransient->uExitIntErrorCode != 0)
6748	return false;
6749
6750	/* Check magic and port. */
6751	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
6752	/Log(("vmxHCIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
6753	if (pCtx->rax != UINT32_C(0x564d5868))
6754	return false;
6755	if (pCtx->dx != UINT32_C(0x5658))
6756	return false;
6757
6758	/* Flat ring-3 CS. */
6759	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
6760	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
6761	/Log(("vmxHCIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
6762	if (pCtx->cs.Attr.n.u2Dpl != 3)
6763	return false;
6764	if (pCtx->cs.u64Base != 0)
6765	return false;
6766
6767	/* Check opcode. */
6768	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
6769	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
6770	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
6771	/Log(("vmxHCIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
6772	if (RT_FAILURE(rc))
6773	return false;
6774	if (abInstr[0] != 0xed)
6775	return false;
6776
6777	return true;
6778	}
6779
6780
6781	/**
6782	* VM-exit exception handler for \#GP (General-protection exception).
6783	*
6784	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
6785	*/
6786	static VBOXSTRICTRC vmxHCExitXcptGP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6787	{
6788	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
6789	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestGP);
6790
6791	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6792	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6793	#ifndef IN_NEM_DARWIN
6794	PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
6795	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
6796	{ /* likely */ }
6797	else
6798	#endif
6799	{
6800	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
6801	# ifndef IN_NEM_DARWIN
6802	Assert(pVCpu->hmr0.s.fUsingDebugLoop \|\| VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
6803	# else
6804	Assert(/pVCpu->hmr0.s.fUsingDebugLoop \|\|/ VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
6805	# endif
6806	#endif
6807	/*
6808	* If the guest is not in real-mode or we have unrestricted guest execution support, or if we are
6809	* executing a nested-guest, reflect #GP to the guest or nested-guest.
6810	*/
6811	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
6812	AssertRCReturn(rc, rc);
6813	Log4Func(("Gst: cs:rip=%#04x:%08RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
6814	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
6815
6816	if ( pVmxTransient->fIsNestedGuest
6817	\|\| !VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv
6818	\|\| !vmxHCIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
6819	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
6820	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
6821	else
6822	rc = vmxHCHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
6823	return rc;
6824	}
6825
6826	#ifndef IN_NEM_DARWIN
6827	Assert(CPUMIsGuestInRealModeEx(pCtx));
6828	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fUnrestrictedGuest);
6829	Assert(!pVmxTransient->fIsNestedGuest);
6830
6831	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
6832	AssertRCReturn(rc, rc);
6833
6834	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
6835	if (rcStrict == VINF_SUCCESS)
6836	{
6837	if (!CPUMIsGuestInRealModeEx(pCtx))
6838	{
6839	/*
6840	* The guest is no longer in real-mode, check if we can continue executing the
6841	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
6842	*/
6843	pVmcsInfoShared->RealMode.fRealOnV86Active = false;
6844	if (HMCanExecuteVmxGuest(pVCpu->CTX_SUFF(pVM), pVCpu, pCtx))
6845	{
6846	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
6847	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
6848	}
6849	else
6850	{
6851	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
6852	rcStrict = VINF_EM_RESCHEDULE;
6853	}
6854	}
6855	else
6856	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
6857	}
6858	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6859	{
6860	rcStrict = VINF_SUCCESS;
6861	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6862	}
6863	return VBOXSTRICTRC_VAL(rcStrict);
6864	#endif
6865	}
6866
6867
6868	/**
6869	* VM-exit exception handler for \#DE (Divide Error).
6870	*
6871	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
6872	*/
6873	static VBOXSTRICTRC vmxHCExitXcptDE(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6874	{
6875	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
6876	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDE);
6877
6878	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
6879	AssertRCReturn(rc, rc);
6880
6881	VBOXSTRICTRC rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
6882	if (VCPU_2_VMXSTATE(pVCpu).fGCMTrapXcptDE)
6883	{
6884	uint8_t cbInstr = 0;
6885	VBOXSTRICTRC rc2 = GCMXcptDE(pVCpu, &pVCpu->cpum.GstCtx, NULL /* pDis */, &cbInstr);
6886	if (rc2 == VINF_SUCCESS)
6887	rcStrict = VINF_SUCCESS; /* Restart instruction with modified guest register context. */
6888	else if (rc2 == VERR_NOT_FOUND)
6889	rcStrict = VERR_NOT_FOUND; /* Deliver the exception. */
6890	else
6891	Assert(RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
6892	}
6893	else
6894	rcStrict = VINF_SUCCESS; /* Do nothing. */
6895
6896	/* If the GCM #DE exception handler didn't succeed or wasn't needed, raise #DE. */
6897	if (RT_FAILURE(rcStrict))
6898	{
6899	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
6900	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
6901	rcStrict = VINF_SUCCESS;
6902	}
6903
6904	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE);
6905	return VBOXSTRICTRC_VAL(rcStrict);
6906	}
6907
6908
6909	/**
6910	* VM-exit exception handler wrapper for all other exceptions that are not handled
6911	* by a specific handler.
6912	*
6913	* This simply re-injects the exception back into the VM without any special
6914	* processing.
6915	*
6916	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
6917	*/
6918	static VBOXSTRICTRC vmxHCExitXcptOthers(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6919	{
6920	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
6921
6922	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
6923	# ifndef IN_NEM_DARWIN
6924	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6925	AssertMsg(pVCpu->hmr0.s.fUsingDebugLoop \|\| pVmcsInfo->pShared->RealMode.fRealOnV86Active \|\| pVmxTransient->fIsNestedGuest,
6926	("uVector=%#x u32XcptBitmap=%#X32\n",
6927	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
6928	NOREF(pVmcsInfo);
6929	# endif
6930	#endif
6931
6932	/*
6933	* Re-inject the exception into the guest. This cannot be a double-fault condition which
6934	* would have been handled while checking exits due to event delivery.
6935	*/
6936	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
6937
6938	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
6939	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6940	AssertRCReturn(rc, rc);
6941	Log4Func(("Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%08RX64\n", uVector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
6942	#endif
6943
6944	#ifdef VBOX_WITH_STATISTICS
6945	switch (uVector)
6946	{
6947	case X86_XCPT_DE: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDE); break;
6948	case X86_XCPT_DB: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDB); break;
6949	case X86_XCPT_BP: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestBP); break;
6950	case X86_XCPT_OF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestOF); break;
6951	case X86_XCPT_BR: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestBR); break;
6952	case X86_XCPT_UD: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestUD); break;
6953	case X86_XCPT_NM: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestOF); break;
6954	case X86_XCPT_DF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDF); break;
6955	case X86_XCPT_TS: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestTS); break;
6956	case X86_XCPT_NP: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestNP); break;
6957	case X86_XCPT_SS: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestSS); break;
6958	case X86_XCPT_GP: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestGP); break;
6959	case X86_XCPT_PF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestPF); break;
6960	case X86_XCPT_MF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestMF); break;
6961	case X86_XCPT_AC: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestAC); break;
6962	case X86_XCPT_XF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestXF); break;
6963	default:
6964	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestXcpUnk);
6965	break;
6966	}
6967	#endif
6968
6969	/* We should never call this function for a page-fault, we'd need to pass on the fault address below otherwise. */
6970	Assert(!VMX_EXIT_INT_INFO_IS_XCPT_PF(pVmxTransient->uExitIntInfo));
6971	NOREF(uVector);
6972
6973	/* Re-inject the original exception into the guest. */
6974	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
6975	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
6976	return VINF_SUCCESS;
6977	}
6978
6979
6980	/**
6981	* VM-exit exception handler for all exceptions (except NMIs!).
6982	*
6983	* @remarks This may be called for both guests and nested-guests. Take care to not
6984	* make assumptions and avoid doing anything that is not relevant when
6985	* executing a nested-guest (e.g., Mesa driver hacks).
6986	*/
6987	static VBOXSTRICTRC vmxHCExitXcpt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6988	{
6989	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
6990
6991	/*
6992	* If this VM-exit occurred while delivering an event through the guest IDT, take
6993	* action based on the return code and additional hints (e.g. for page-faults)
6994	* that will be updated in the VMX transient structure.
6995	*/
6996	VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
6997	if (rcStrict == VINF_SUCCESS)
6998	{
6999	/*
7000	* If an exception caused a VM-exit due to delivery of an event, the original
7001	* event may have to be re-injected into the guest. We shall reinject it and
7002	* continue guest execution. However, page-fault is a complicated case and
7003	* needs additional processing done in vmxHCExitXcptPF().
7004	*/
7005	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
7006	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
7007	if ( !VCPU_2_VMXSTATE(pVCpu).Event.fPending
7008	\|\| uVector == X86_XCPT_PF)
7009	{
7010	switch (uVector)
7011	{
7012	case X86_XCPT_PF: return vmxHCExitXcptPF(pVCpu, pVmxTransient);
7013	case X86_XCPT_GP: return vmxHCExitXcptGP(pVCpu, pVmxTransient);
7014	case X86_XCPT_MF: return vmxHCExitXcptMF(pVCpu, pVmxTransient);
7015	case X86_XCPT_DB: return vmxHCExitXcptDB(pVCpu, pVmxTransient);
7016	case X86_XCPT_BP: return vmxHCExitXcptBP(pVCpu, pVmxTransient);
7017	case X86_XCPT_AC: return vmxHCExitXcptAC(pVCpu, pVmxTransient);
7018	case X86_XCPT_DE: return vmxHCExitXcptDE(pVCpu, pVmxTransient);
7019	default:
7020	return vmxHCExitXcptOthers(pVCpu, pVmxTransient);
7021	}
7022	}
7023	/* else: inject pending event before resuming guest execution. */
7024	}
7025	else if (rcStrict == VINF_HM_DOUBLE_FAULT)
7026	{
7027	Assert(VCPU_2_VMXSTATE(pVCpu).Event.fPending);
7028	rcStrict = VINF_SUCCESS;
7029	}
7030
7031	return rcStrict;
7032	}
7033	/** @} */
7034
7035
7036	/** @name VM-exit handlers.
7037	* @{
7038	*/
7039	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7040	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
7041	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7042
7043	/**
7044	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
7045	*/
7046	HMVMX_EXIT_DECL vmxHCExitExtInt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7047	{
7048	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7049	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitExtInt);
7050
7051	#ifndef IN_NEM_DARWIN
7052	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
7053	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
7054	return VINF_SUCCESS;
7055	return VINF_EM_RAW_INTERRUPT;
7056	#else
7057	return VINF_SUCCESS;
7058	#endif
7059	}
7060
7061
7062	/**
7063	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
7064	* VM-exit.
7065	*/
7066	HMVMX_EXIT_DECL vmxHCExitXcptOrNmi(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7067	{
7068	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7069	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitXcptNmi, y3);
7070
7071	vmxHCReadToTransient<HMVMX_READ_EXIT_INTERRUPTION_INFO>(pVCpu, pVmxTransient);
7072
7073	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
7074	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
7075	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
7076
7077	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7078	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
7079	&& uExitIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
7080	NOREF(pVmcsInfo);
7081
7082	VBOXSTRICTRC rcStrict;
7083	switch (uExitIntType)
7084	{
7085	#ifndef IN_NEM_DARWIN /* NMIs should never reach R3. */
7086	/*
7087	* Host physical NMIs:
7088	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
7089	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
7090	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
7091	*
7092	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
7093	* See Intel spec. 27.5.5 "Updating Non-Register State".
7094	*/
7095	case VMX_EXIT_INT_INFO_TYPE_NMI:
7096	{
7097	rcStrict = hmR0VmxExitHostNmi(pVCpu, pVmcsInfo);
7098	break;
7099	}
7100	#endif
7101
7102	/*
7103	* Privileged software exceptions (#DB from ICEBP),
7104	* Software exceptions (#BP and #OF),
7105	* Hardware exceptions:
7106	* Process the required exceptions and resume guest execution if possible.
7107	*/
7108	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
7109	Assert(uVector == X86_XCPT_DB);
7110	RT_FALL_THRU();
7111	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
7112	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uExitIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
7113	RT_FALL_THRU();
7114	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
7115	{
7116	NOREF(uVector);
7117	vmxHCReadToTransient< HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
7118	\| HMVMX_READ_EXIT_INSTR_LEN
7119	\| HMVMX_READ_IDT_VECTORING_INFO
7120	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);
7121	rcStrict = vmxHCExitXcpt(pVCpu, pVmxTransient);
7122	break;
7123	}
7124
7125	default:
7126	{
7127	VCPU_2_VMXSTATE(pVCpu).u32HMError = pVmxTransient->uExitIntInfo;
7128	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
7129	AssertMsgFailed(("Invalid/unexpected VM-exit interruption info %#x\n", pVmxTransient->uExitIntInfo));
7130	break;
7131	}
7132	}
7133
7134	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitXcptNmi, y3);
7135	return rcStrict;
7136	}
7137
7138
7139	/**
7140	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7141	*/
7142	HMVMX_EXIT_NSRC_DECL vmxHCExitIntWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7143	{
7144	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7145
7146	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7147	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7148	vmxHCClearIntWindowExitVmcs(pVCpu, pVmcsInfo);
7149
7150	/* Evaluate and deliver pending events and resume guest execution. */
7151	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitIntWindow);
7152	return VINF_SUCCESS;
7153	}
7154
7155
7156	/**
7157	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7158	*/
7159	HMVMX_EXIT_NSRC_DECL vmxHCExitNmiWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7160	{
7161	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7162
7163	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7164	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
7165	{
7166	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7167	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
7168	}
7169
7170	Assert(!CPUMIsGuestNmiBlocking(pVCpu));
7171
7172	/*
7173	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
7174	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
7175	*/
7176	uint32_t fIntrState;
7177	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
7178	AssertRC(rc);
7179	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
7180	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
7181	{
7182	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7183	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7184
7185	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
7186	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, fIntrState);
7187	AssertRC(rc);
7188	}
7189
7190	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
7191	vmxHCClearNmiWindowExitVmcs(pVCpu, pVmcsInfo);
7192
7193	/* Evaluate and deliver pending events and resume guest execution. */
7194	return VINF_SUCCESS;
7195	}
7196
7197
7198	/**
7199	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7200	*/
7201	HMVMX_EXIT_NSRC_DECL vmxHCExitWbinvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7202	{
7203	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7204	return vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
7205	}
7206
7207
7208	/**
7209	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7210	*/
7211	HMVMX_EXIT_NSRC_DECL vmxHCExitInvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7212	{
7213	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7214	return vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
7215	}
7216
7217
7218	/**
7219	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7220	*/
7221	HMVMX_EXIT_DECL vmxHCExitCpuid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7222	{
7223	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7224
7225	/*
7226	* Get the state we need and update the exit history entry.
7227	*/
7228	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7229	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
7230	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
7231	AssertRCReturn(rc, rc);
7232
7233	VBOXSTRICTRC rcStrict;
7234	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
7235	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
7236	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
7237	if (!pExitRec)
7238	{
7239	/*
7240	* Regular CPUID instruction execution.
7241	*/
7242	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbExitInstr);
7243	if (rcStrict == VINF_SUCCESS)
7244	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
7245	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7246	{
7247	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7248	rcStrict = VINF_SUCCESS;
7249	}
7250	}
7251	else
7252	{
7253	/*
7254	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
7255	*/
7256	int rc2 = vmxHCImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
7257	AssertRCReturn(rc2, rc2);
7258
7259	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
7260	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
7261
7262	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
7263	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
7264
7265	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
7266	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
7267	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
7268	}
7269	return rcStrict;
7270	}
7271
7272
7273	/**
7274	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7275	*/
7276	HMVMX_EXIT_DECL vmxHCExitGetsec(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7277	{
7278	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7279
7280	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7281	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
7282	AssertRCReturn(rc, rc);
7283
7284	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
7285	return VINF_EM_RAW_EMULATE_INSTR;
7286
7287	AssertMsgFailed(("vmxHCExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
7288	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
7289	}
7290
7291
7292	/**
7293	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7294	*/
7295	HMVMX_EXIT_DECL vmxHCExitRdtsc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7296	{
7297	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7298
7299	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7300	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
7301	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
7302	AssertRCReturn(rc, rc);
7303
7304	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbExitInstr);
7305	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
7306	{
7307	/* If we get a spurious VM-exit when TSC offsetting is enabled,
7308	we must reset offsetting on VM-entry. See @bugref{6634}. */
7309	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
7310	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
7311	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
7312	}
7313	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7314	{
7315	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7316	rcStrict = VINF_SUCCESS;
7317	}
7318	return rcStrict;
7319	}
7320
7321
7322	/**
7323	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7324	*/
7325	HMVMX_EXIT_DECL vmxHCExitRdtscp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7326	{
7327	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7328
7329	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7330	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
7331	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
7332	AssertRCReturn(rc, rc);
7333
7334	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbExitInstr);
7335	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
7336	{
7337	/* If we get a spurious VM-exit when TSC offsetting is enabled,
7338	we must reset offsetting on VM-reentry. See @bugref{6634}. */
7339	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
7340	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
7341	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
7342	}
7343	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7344	{
7345	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7346	rcStrict = VINF_SUCCESS;
7347	}
7348	return rcStrict;
7349	}
7350
7351
7352	/**
7353	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7354	*/
7355	HMVMX_EXIT_DECL vmxHCExitRdpmc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7356	{
7357	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7358
7359	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7360	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
7361	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
7362	AssertRCReturn(rc, rc);
7363
7364	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7365	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
7366	if (RT_LIKELY(rc == VINF_SUCCESS))
7367	{
7368	rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
7369	Assert(pVmxTransient->cbExitInstr == 2);
7370	}
7371	else
7372	{
7373	AssertMsgFailed(("vmxHCExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7374	rc = VERR_EM_INTERPRETER;
7375	}
7376	return rc;
7377	}
7378
7379
7380	/**
7381	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
7382	*/
7383	HMVMX_EXIT_DECL vmxHCExitVmcall(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7384	{
7385	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7386
7387	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
7388	if (EMAreHypercallInstructionsEnabled(pVCpu))
7389	{
7390	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7391	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
7392	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
7393	AssertRCReturn(rc, rc);
7394
7395	/* Perform the hypercall. */
7396	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
7397	if (rcStrict == VINF_SUCCESS)
7398	{
7399	rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
7400	AssertRCReturn(rc, rc);
7401	}
7402	else
7403	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
7404	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
7405	\|\| RT_FAILURE(rcStrict));
7406
7407	/* If the hypercall changes anything other than guest's general-purpose registers,
7408	we would need to reload the guest changed bits here before VM-entry. */
7409	}
7410	else
7411	Log4Func(("Hypercalls not enabled\n"));
7412
7413	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
7414	if (RT_FAILURE(rcStrict))
7415	{
7416	vmxHCSetPendingXcptUD(pVCpu);
7417	rcStrict = VINF_SUCCESS;
7418	}
7419
7420	return rcStrict;
7421	}
7422
7423
7424	/**
7425	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7426	*/
7427	HMVMX_EXIT_DECL vmxHCExitInvlpg(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7428	{
7429	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7430	#ifndef IN_NEM_DARWIN
7431	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging \|\| pVCpu->hmr0.s.fUsingDebugLoop);
7432	#endif
7433
7434	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7435	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
7436	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
7437	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
7438	AssertRCReturn(rc, rc);
7439
7440	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->uExitQual);
7441
7442	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
7443	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
7444	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7445	{
7446	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7447	rcStrict = VINF_SUCCESS;
7448	}
7449	else
7450	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
7451	VBOXSTRICTRC_VAL(rcStrict)));
7452	return rcStrict;
7453	}
7454
7455
7456	/**
7457	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7458	*/
7459	HMVMX_EXIT_DECL vmxHCExitMonitor(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7460	{
7461	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7462
7463	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7464	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
7465	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
7466	AssertRCReturn(rc, rc);
7467
7468	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbExitInstr);
7469	if (rcStrict == VINF_SUCCESS)
7470	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
7471	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7472	{
7473	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7474	rcStrict = VINF_SUCCESS;
7475	}
7476
7477	return rcStrict;
7478	}
7479
7480
7481	/**
7482	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7483	*/
7484	HMVMX_EXIT_DECL vmxHCExitMwait(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7485	{
7486	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7487
7488	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7489	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
7490	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
7491	AssertRCReturn(rc, rc);
7492
7493	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbExitInstr);
7494	if (RT_SUCCESS(rcStrict))
7495	{
7496	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
7497	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
7498	rcStrict = VINF_SUCCESS;
7499	}
7500
7501	return rcStrict;
7502	}
7503
7504
7505	/**
7506	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7507	* VM-exit.
7508	*/
7509	HMVMX_EXIT_DECL vmxHCExitTripleFault(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7510	{
7511	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7512	return VINF_EM_RESET;
7513	}
7514
7515
7516	/**
7517	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7518	*/
7519	HMVMX_EXIT_DECL vmxHCExitHlt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7520	{
7521	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7522
7523	int rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
7524	AssertRCReturn(rc, rc);
7525
7526	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
7527	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
7528	rc = VINF_SUCCESS;
7529	else
7530	rc = VINF_EM_HALT;
7531
7532	if (rc != VINF_SUCCESS)
7533	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchHltToR3);
7534	return rc;
7535	}
7536
7537
7538	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
7539	/**
7540	* VM-exit handler for instructions that result in a \#UD exception delivered to
7541	* the guest.
7542	*/
7543	HMVMX_EXIT_NSRC_DECL vmxHCExitSetPendingXcptUD(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7544	{
7545	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7546	vmxHCSetPendingXcptUD(pVCpu);
7547	return VINF_SUCCESS;
7548	}
7549	#endif
7550
7551
7552	/**
7553	* VM-exit handler for expiry of the VMX-preemption timer.
7554	*/
7555	HMVMX_EXIT_DECL vmxHCExitPreemptTimer(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7556	{
7557	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7558
7559	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
7560	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
7561	Log12(("vmxHCExitPreemptTimer:\n"));
7562
7563	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7564	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7565	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7566	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitPreemptTimer);
7567	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7568	}
7569
7570
7571	/**
7572	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7573	*/
7574	HMVMX_EXIT_DECL vmxHCExitXsetbv(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7575	{
7576	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7577
7578	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7579	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
7580	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
7581	AssertRCReturn(rc, rc);
7582
7583	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbExitInstr);
7584	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
7585	: HM_CHANGED_RAISED_XCPT_MASK);
7586
7587	#ifndef IN_NEM_DARWIN
7588	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7589	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
7590	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
7591	{
7592	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
7593	hmR0VmxUpdateStartVmFunction(pVCpu);
7594	}
7595	#endif
7596
7597	return rcStrict;
7598	}
7599
7600
7601	/**
7602	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7603	*/
7604	HMVMX_EXIT_DECL vmxHCExitInvpcid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7605	{
7606	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7607
7608	/** @todo Enable the new code after finding a reliably guest test-case. */
7609	#if 1
7610	return VERR_EM_INTERPRETER;
7611	#else
7612	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
7613	\| HMVMX_READ_EXIT_INSTR_INFO
7614	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
7615	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
7616	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
7617	AssertRCReturn(rc, rc);
7618
7619	/* Paranoia. Ensure this has a memory operand. */
7620	Assert(!pVmxTransient->ExitInstrInfo.Inv.u1Cleared0);
7621
7622	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
7623	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
7624	uint64_t const uType = CPUMIsGuestIn64BitCode(pVCpu) ? pVCpu->cpum.GstCtx.aGRegs[iGReg].u64
7625	: pVCpu->cpum.GstCtx.aGRegs[iGReg].u32;
7626
7627	RTGCPTR GCPtrDesc;
7628	HMVMX_DECODE_MEM_OPERAND(pVCpu, pVmxTransient->ExitInstrInfo.u, pVmxTransient->uExitQual, VMXMEMACCESS_READ, &GCPtrDesc);
7629
7630	VBOXSTRICTRC rcStrict = IEMExecDecodedInvpcid(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->ExitInstrInfo.Inv.iSegReg,
7631	GCPtrDesc, uType);
7632	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
7633	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
7634	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7635	{
7636	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7637	rcStrict = VINF_SUCCESS;
7638	}
7639	return rcStrict;
7640	#endif
7641	}
7642
7643
7644	/**
7645	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
7646	* VM-exit.
7647	*/
7648	HMVMX_EXIT_NSRC_DECL vmxHCExitErrInvalidGuestState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7649	{
7650	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7651	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
7652	AssertRCReturn(rc, rc);
7653
7654	rc = vmxHCCheckCachedVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
7655	if (RT_FAILURE(rc))
7656	return rc;
7657
7658	uint32_t const uInvalidReason = vmxHCCheckGuestState(pVCpu, pVmcsInfo);
7659	NOREF(uInvalidReason);
7660
7661	#ifdef VBOX_STRICT
7662	uint32_t fIntrState;
7663	uint64_t u64Val;
7664	vmxHCReadToTransient< HMVMX_READ_EXIT_INSTR_INFO
7665	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
7666	vmxHCReadEntryXcptErrorCodeVmcs(pVCpu, pVmxTransient);
7667
7668	Log4(("uInvalidReason %u\n", uInvalidReason));
7669	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
7670	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7671	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7672
7673	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &fIntrState); AssertRC(rc);
7674	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
7675	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
7676	Log4(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
7677	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_MASK, &u64Val); AssertRC(rc);
7678	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RX64\n", u64Val));
7679	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Val); AssertRC(rc);
7680	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RX64\n", u64Val));
7681	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_MASK, &u64Val); AssertRC(rc);
7682	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RX64\n", u64Val));
7683	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Val); AssertRC(rc);
7684	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RX64\n", u64Val));
7685	# ifndef IN_NEM_DARWIN
7686	if (pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
7687	{
7688	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7689	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7690	}
7691
7692	hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
7693	# endif
7694	#endif
7695
7696	return VERR_VMX_INVALID_GUEST_STATE;
7697	}
7698
7699	/**
7700	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
7701	*/
7702	HMVMX_EXIT_NSRC_DECL vmxHCExitErrUnexpected(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7703	{
7704	/*
7705	* Cumulative notes of all recognized but unexpected VM-exits.
7706	*
7707	* 1. This does -not- cover scenarios like a page-fault VM-exit occurring when
7708	* nested-paging is used.
7709	*
7710	* 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
7711	* emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
7712	* this function (and thereby stop VM execution) for handling such instructions.
7713	*
7714	*
7715	* VMX_EXIT_INIT_SIGNAL:
7716	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
7717	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
7718	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
7719	*
7720	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
7721	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
7722	* See Intel spec. "23.8 Restrictions on VMX operation".
7723	*
7724	* VMX_EXIT_SIPI:
7725	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
7726	* activity state is used. We don't make use of it as our guests don't have direct
7727	* access to the host local APIC.
7728	*
7729	* See Intel spec. 25.3 "Other Causes of VM-exits".
7730	*
7731	* VMX_EXIT_IO_SMI:
7732	* VMX_EXIT_SMI:
7733	* This can only happen if we support dual-monitor treatment of SMI, which can be
7734	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
7735	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
7736	* VMX root mode or receive an SMI. If we get here, something funny is going on.
7737	*
7738	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
7739	* See Intel spec. 25.3 "Other Causes of VM-Exits"
7740	*
7741	* VMX_EXIT_ERR_MSR_LOAD:
7742	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
7743	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
7744	* execution.
7745	*
7746	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
7747	*
7748	* VMX_EXIT_ERR_MACHINE_CHECK:
7749	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
7750	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
7751	* #MC exception abort class exception is raised. We thus cannot assume a
7752	* reasonable chance of continuing any sort of execution and we bail.
7753	*
7754	* See Intel spec. 15.1 "Machine-check Architecture".
7755	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
7756	*
7757	* VMX_EXIT_PML_FULL:
7758	* VMX_EXIT_VIRTUALIZED_EOI:
7759	* VMX_EXIT_APIC_WRITE:
7760	* We do not currently support any of these features and thus they are all unexpected
7761	* VM-exits.
7762	*
7763	* VMX_EXIT_GDTR_IDTR_ACCESS:
7764	* VMX_EXIT_LDTR_TR_ACCESS:
7765	* VMX_EXIT_RDRAND:
7766	* VMX_EXIT_RSM:
7767	* VMX_EXIT_VMFUNC:
7768	* VMX_EXIT_ENCLS:
7769	* VMX_EXIT_RDSEED:
7770	* VMX_EXIT_XSAVES:
7771	* VMX_EXIT_XRSTORS:
7772	* VMX_EXIT_UMWAIT:
7773	* VMX_EXIT_TPAUSE:
7774	* VMX_EXIT_LOADIWKEY:
7775	* These VM-exits are -not- caused unconditionally by execution of the corresponding
7776	* instruction. Any VM-exit for these instructions indicate a hardware problem,
7777	* unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
7778	*
7779	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
7780	*/
7781	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7782	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
7783	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
7784	}
7785
7786
7787	/**
7788	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7789	*/
7790	HMVMX_EXIT_DECL vmxHCExitRdmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7791	{
7792	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7793
7794	/** @todo Optimize this: We currently drag in the whole MSR state
7795	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
7796	* MSRs required. That would require changes to IEM and possibly CPUM too.
7797	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
7798	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7799	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
7800	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
7801	switch (idMsr)
7802	{
7803	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
7804	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
7805	}
7806
7807	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
7808	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, fImport);
7809	AssertRCReturn(rc, rc);
7810
7811	Log4Func(("ecx=%#RX32\n", idMsr));
7812
7813	#if defined(VBOX_STRICT) && !defined(IN_NEM_DARWIN)
7814	Assert(!pVmxTransient->fIsNestedGuest);
7815	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
7816	{
7817	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
7818	&& idMsr != MSR_K6_EFER)
7819	{
7820	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
7821	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
7822	}
7823	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
7824	{
7825	Assert(pVmcsInfo->pvMsrBitmap);
7826	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
7827	if (fMsrpm & VMXMSRPM_ALLOW_RD)
7828	{
7829	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
7830	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
7831	}
7832	}
7833	}
7834	#endif
7835
7836	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbExitInstr);
7837	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitRdmsr);
7838	if (rcStrict == VINF_SUCCESS)
7839	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
7840	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7841	{
7842	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7843	rcStrict = VINF_SUCCESS;
7844	}
7845	else
7846	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ \|\| rcStrict == VINF_EM_TRIPLE_FAULT,
7847	("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
7848
7849	return rcStrict;
7850	}
7851
7852
7853	/**
7854	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7855	*/
7856	HMVMX_EXIT_DECL vmxHCExitWrmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7857	{
7858	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7859
7860	/** @todo Optimize this: We currently drag in the whole MSR state
7861	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
7862	* MSRs required. That would require changes to IEM and possibly CPUM too.
7863	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
7864	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
7865	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
7866
7867	/*
7868	* The FS and GS base MSRs are not part of the above all-MSRs mask.
7869	* Although we don't need to fetch the base as it will be overwritten shortly, while
7870	* loading guest-state we would also load the entire segment register including limit
7871	* and attributes and thus we need to load them here.
7872	*/
7873	switch (idMsr)
7874	{
7875	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
7876	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
7877	}
7878
7879	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7880	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
7881	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, fImport);
7882	AssertRCReturn(rc, rc);
7883
7884	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
7885
7886	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbExitInstr);
7887	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitWrmsr);
7888
7889	if (rcStrict == VINF_SUCCESS)
7890	{
7891	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
7892
7893	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7894	if ( idMsr == MSR_IA32_APICBASE
7895	\|\| ( idMsr >= MSR_IA32_X2APIC_START
7896	&& idMsr <= MSR_IA32_X2APIC_END))
7897	{
7898	/*
7899	* We've already saved the APIC related guest-state (TPR) in post-run phase.
7900	* When full APIC register virtualization is implemented we'll have to make
7901	* sure APIC state is saved from the VMCS before IEM changes it.
7902	*/
7903	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7904	}
7905	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
7906	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
7907	else if (idMsr == MSR_K6_EFER)
7908	{
7909	/*
7910	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
7911	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
7912	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
7913	*/
7914	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
7915	}
7916
7917	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not used. */
7918	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
7919	{
7920	switch (idMsr)
7921	{
7922	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
7923	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
7924	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
7925	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_FS); break;
7926	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_GS); break;
7927	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
7928	default:
7929	{
7930	#ifndef IN_NEM_DARWIN
7931	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
7932	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
7933	else if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
7934	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
7935	#else
7936	AssertMsgFailed(("TODO\n"));
7937	#endif
7938	break;
7939	}
7940	}
7941	}
7942	#if defined(VBOX_STRICT) && !defined(IN_NEM_DARWIN)
7943	else
7944	{
7945	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7946	switch (idMsr)
7947	{
7948	case MSR_IA32_SYSENTER_CS:
7949	case MSR_IA32_SYSENTER_EIP:
7950	case MSR_IA32_SYSENTER_ESP:
7951	case MSR_K8_FS_BASE:
7952	case MSR_K8_GS_BASE:
7953	{
7954	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
7955	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
7956	}
7957
7958	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
7959	default:
7960	{
7961	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
7962	{
7963	/* EFER MSR writes are always intercepted. */
7964	if (idMsr != MSR_K6_EFER)
7965	{
7966	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
7967	idMsr));
7968	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
7969	}
7970	}
7971
7972	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
7973	{
7974	Assert(pVmcsInfo->pvMsrBitmap);
7975	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
7976	if (fMsrpm & VMXMSRPM_ALLOW_WR)
7977	{
7978	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
7979	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
7980	}
7981	}
7982	break;
7983	}
7984	}
7985	}
7986	#endif /* VBOX_STRICT */
7987	}
7988	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7989	{
7990	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7991	rcStrict = VINF_SUCCESS;
7992	}
7993	else
7994	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE \|\| rcStrict == VINF_EM_TRIPLE_FAULT,
7995	("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
7996
7997	return rcStrict;
7998	}
7999
8000
8001	/**
8002	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
8003	*/
8004	HMVMX_EXIT_DECL vmxHCExitPause(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8005	{
8006	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8007
8008	/** @todo The guest has likely hit a contended spinlock. We might want to
8009	* poke a schedule different guest VCPU. */
8010	int rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
8011	if (RT_SUCCESS(rc))
8012	return VINF_EM_RAW_INTERRUPT;
8013
8014	AssertMsgFailed(("vmxHCExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
8015	return rc;
8016	}
8017
8018
8019	/**
8020	* VM-exit handler for when the TPR value is lowered below the specified
8021	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
8022	*/
8023	HMVMX_EXIT_NSRC_DECL vmxHCExitTprBelowThreshold(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8024	{
8025	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8026	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
8027
8028	/*
8029	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
8030	* We'll re-evaluate pending interrupts and inject them before the next VM
8031	* entry so we can just continue execution here.
8032	*/
8033	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitTprBelowThreshold);
8034	return VINF_SUCCESS;
8035	}
8036
8037
8038	/**
8039	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
8040	* VM-exit.
8041	*
8042	* @retval VINF_SUCCESS when guest execution can continue.
8043	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
8044	* @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
8045	* incompatible guest state for VMX execution (real-on-v86 case).
8046	*/
8047	HMVMX_EXIT_DECL vmxHCExitMovCRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8048	{
8049	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8050	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitMovCRx, y2);
8051
8052	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8053	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
8054	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
8055
8056	VBOXSTRICTRC rcStrict;
8057	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8058	uint64_t const uExitQual = pVmxTransient->uExitQual;
8059	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
8060	switch (uAccessType)
8061	{
8062	/*
8063	* MOV to CRx.
8064	*/
8065	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
8066	{
8067	/*
8068	* When PAE paging is used, the CPU will reload PAE PDPTEs from CR3 when the guest
8069	* changes certain bits even in CR0, CR4 (and not just CR3). We are currently fine
8070	* since IEM_CPUMCTX_EXTRN_MUST_MASK (used below) includes CR3 which will import
8071	* PAE PDPTEs as well.
8072	*/
8073	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
8074	AssertRCReturn(rc, rc);
8075
8076	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
8077	#ifndef IN_NEM_DARWIN
8078	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
8079	#endif
8080	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
8081	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
8082
8083	/*
8084	* MOV to CR3 only cause a VM-exit when one or more of the following are true:
8085	* - When nested paging isn't used.
8086	* - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
8087	* - We are executing in the VM debug loop.
8088	*/
8089	#ifndef HMVMX_ALWAYS_INTERCEPT_CR3_ACCESS
8090	# ifndef IN_NEM_DARWIN
8091	Assert( iCrReg != 3
8092	\|\| !VM_IS_VMX_NESTED_PAGING(pVM)
8093	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
8094	\|\| pVCpu->hmr0.s.fUsingDebugLoop);
8095	# else
8096	Assert( iCrReg != 3
8097	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx));
8098	# endif
8099	#endif
8100
8101	/* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
8102	Assert( iCrReg != 8
8103	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
8104
8105	rcStrict = vmxHCExitMovToCrX(pVCpu, pVmxTransient->cbExitInstr, iGReg, iCrReg);
8106	AssertMsg( rcStrict == VINF_SUCCESS
8107	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8108
8109	#ifndef IN_NEM_DARWIN
8110	/*
8111	* This is a kludge for handling switches back to real mode when we try to use
8112	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
8113	* deal with special selector values, so we have to return to ring-3 and run
8114	* there till the selector values are V86 mode compatible.
8115	*
8116	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
8117	* latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
8118	* this function.
8119	*/
8120	if ( iCrReg == 0
8121	&& rcStrict == VINF_SUCCESS
8122	&& !VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
8123	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
8124	&& (uOldCr0 & X86_CR0_PE)
8125	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
8126	{
8127	/** @todo Check selectors rather than returning all the time. */
8128	Assert(!pVmxTransient->fIsNestedGuest);
8129	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
8130	rcStrict = VINF_EM_RESCHEDULE_REM;
8131	}
8132	#endif
8133
8134	break;
8135	}
8136
8137	/*
8138	* MOV from CRx.
8139	*/
8140	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
8141	{
8142	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
8143	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
8144
8145	/*
8146	* MOV from CR3 only cause a VM-exit when one or more of the following are true:
8147	* - When nested paging isn't used.
8148	* - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
8149	* - We are executing in the VM debug loop.
8150	*/
8151	#ifndef HMVMX_ALWAYS_INTERCEPT_CR3_ACCESS
8152	# ifndef IN_NEM_DARWIN
8153	Assert( iCrReg != 3
8154	\|\| !VM_IS_VMX_NESTED_PAGING(pVM)
8155	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
8156	\|\| pVCpu->hmr0.s.fLeaveDone);
8157	# else
8158	Assert( iCrReg != 3
8159	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx));
8160	# endif
8161	#endif
8162
8163	/* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
8164	Assert( iCrReg != 8
8165	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
8166
8167	rcStrict = vmxHCExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
8168	break;
8169	}
8170
8171	/*
8172	* CLTS (Clear Task-Switch Flag in CR0).
8173	*/
8174	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
8175	{
8176	rcStrict = vmxHCExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr);
8177	break;
8178	}
8179
8180	/*
8181	* LMSW (Load Machine-Status Word into CR0).
8182	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
8183	*/
8184	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
8185	{
8186	RTGCPTR GCPtrEffDst;
8187	uint8_t const cbInstr = pVmxTransient->cbExitInstr;
8188	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
8189	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
8190	if (fMemOperand)
8191	{
8192	vmxHCReadToTransient<HMVMX_READ_GUEST_LINEAR_ADDR>(pVCpu, pVmxTransient);
8193	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
8194	}
8195	else
8196	GCPtrEffDst = NIL_RTGCPTR;
8197	rcStrict = vmxHCExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
8198	break;
8199	}
8200
8201	default:
8202	{
8203	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
8204	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
8205	}
8206	}
8207
8208	Assert((VCPU_2_VMXSTATE(pVCpu).fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
8209	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
8210	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
8211
8212	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitMovCRx, y2);
8213	NOREF(pVM);
8214	return rcStrict;
8215	}
8216
8217
8218	/**
8219	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
8220	* VM-exit.
8221	*/
8222	HMVMX_EXIT_DECL vmxHCExitIoInstr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8223	{
8224	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8225	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitIO, y1);
8226
8227	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8228	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8229	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
8230	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
8231	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
8232	\| CPUMCTX_EXTRN_EFER);
8233	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
8234	AssertRCReturn(rc, rc);
8235
8236	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
8237	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
8238	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
8239	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
8240	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
8241	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
8242	bool const fDbgStepping = VCPU_2_VMXSTATE(pVCpu).fSingleInstruction;
8243	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
8244
8245	/*
8246	* Update exit history to see if this exit can be optimized.
8247	*/
8248	VBOXSTRICTRC rcStrict;
8249	PCEMEXITREC pExitRec = NULL;
8250	if ( !fGstStepping
8251	&& !fDbgStepping)
8252	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
8253	!fIOString
8254	? !fIOWrite
8255	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
8256	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
8257	: !fIOWrite
8258	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
8259	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
8260	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
8261	if (!pExitRec)
8262	{
8263	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
8264	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
8265
8266	uint32_t const cbValue = s_aIOSizes[uIOSize];
8267	uint32_t const cbInstr = pVmxTransient->cbExitInstr;
8268	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
8269	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8270	if (fIOString)
8271	{
8272	/*
8273	* INS/OUTS - I/O String instruction.
8274	*
8275	* Use instruction-information if available, otherwise fall back on
8276	* interpreting the instruction.
8277	*/
8278	Log4Func(("cs:rip=%#04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
8279	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
8280	bool const fInsOutsInfo = RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
8281	if (fInsOutsInfo)
8282	{
8283	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_INFO>(pVCpu, pVmxTransient);
8284	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
8285	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
8286	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
8287	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
8288	if (fIOWrite)
8289	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
8290	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
8291	else
8292	{
8293	/*
8294	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
8295	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
8296	* See Intel Instruction spec. for "INS".
8297	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
8298	*/
8299	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
8300	}
8301	}
8302	else
8303	rcStrict = IEMExecOne(pVCpu);
8304
8305	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP);
8306	fUpdateRipAlready = true;
8307	}
8308	else
8309	{
8310	/*
8311	* IN/OUT - I/O instruction.
8312	*/
8313	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
8314	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
8315	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
8316	if (fIOWrite)
8317	{
8318	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
8319	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitIOWrite);
8320	#ifndef IN_NEM_DARWIN
8321	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
8322	&& !pCtx->eflags.Bits.u1TF)
8323	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
8324	#endif
8325	}
8326	else
8327	{
8328	uint32_t u32Result = 0;
8329	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
8330	if (IOM_SUCCESS(rcStrict))
8331	{
8332	/* Save result of I/O IN instr. in AL/AX/EAX. */
8333	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
8334	}
8335	#ifndef IN_NEM_DARWIN
8336	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
8337	&& !pCtx->eflags.Bits.u1TF)
8338	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
8339	#endif
8340	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitIORead);
8341	}
8342	}
8343
8344	if (IOM_SUCCESS(rcStrict))
8345	{
8346	if (!fUpdateRipAlready)
8347	{
8348	vmxHCAdvanceGuestRipBy(pVCpu, cbInstr);
8349	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP);
8350	}
8351
8352	/*
8353	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
8354	* while booting Fedora 17 64-bit guest.
8355	*
8356	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
8357	*/
8358	if (fIOString)
8359	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
8360
8361	/*
8362	* If any I/O breakpoints are armed, we need to check if one triggered
8363	* and take appropriate action.
8364	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
8365	*/
8366	rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
8367	AssertRCReturn(rc, rc);
8368
8369	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
8370	* execution engines about whether hyper BPs and such are pending. */
8371	uint32_t const uDr7 = pCtx->dr[7];
8372	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
8373	&& X86_DR7_ANY_RW_IO(uDr7)
8374	&& (pCtx->cr4 & X86_CR4_DE))
8375	\|\| DBGFBpIsHwIoArmed(pVM)))
8376	{
8377	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatDRxIoCheck);
8378
8379	#ifndef IN_NEM_DARWIN
8380	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
8381	VMMRZCallRing3Disable(pVCpu);
8382	HM_DISABLE_PREEMPT(pVCpu);
8383
8384	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
8385
8386	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
8387	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
8388	{
8389	/* Raise #DB. */
8390	if (fIsGuestDbgActive)
8391	ASMSetDR6(pCtx->dr[6]);
8392	if (pCtx->dr[7] != uDr7)
8393	VCPU_2_VMXSTATE(pVCpu).fCtxChanged \|= HM_CHANGED_GUEST_DR7;
8394
8395	vmxHCSetPendingXcptDB(pVCpu);
8396	}
8397	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
8398	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
8399	else if ( rcStrict2 != VINF_SUCCESS
8400	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
8401	rcStrict = rcStrict2;
8402	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
8403
8404	HM_RESTORE_PREEMPT();
8405	VMMRZCallRing3Enable(pVCpu);
8406	#else
8407	/** @todo */
8408	#endif
8409	}
8410	}
8411
8412	#ifdef VBOX_STRICT
8413	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
8414	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
8415	Assert(!fIOWrite);
8416	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
8417	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
8418	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
8419	Assert(fIOWrite);
8420	else
8421	{
8422	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
8423	* statuses, that the VMM device and some others may return. See
8424	* IOM_SUCCESS() for guidance. */
8425	AssertMsg( RT_FAILURE(rcStrict)
8426	\|\| rcStrict == VINF_SUCCESS
8427	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
8428	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
8429	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
8430	\|\| rcStrict == VINF_EM_RAW_TO_R3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8431	# endif
8432	}
8433	#endif
8434	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitIO, y1);
8435	}
8436	else
8437	{
8438	/*
8439	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
8440	*/
8441	int rc2 = vmxHCImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8442	AssertRCReturn(rc2, rc2);
8443	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &VCPU_2_VMXSTATS(pVCpu).StatExitIOWrite : &VCPU_2_VMXSTATS(pVCpu).StatExitIORead
8444	: fIOWrite ? &VCPU_2_VMXSTATS(pVCpu).StatExitIOStringWrite : &VCPU_2_VMXSTATS(pVCpu).StatExitIOStringRead);
8445	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
8446	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
8447	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
8448	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
8449
8450	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
8451	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
8452
8453	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
8454	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
8455	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
8456	}
8457	return rcStrict;
8458	}
8459
8460
8461	/**
8462	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8463	* VM-exit.
8464	*/
8465	HMVMX_EXIT_DECL vmxHCExitTaskSwitch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8466	{
8467	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8468
8469	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8470	vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
8471	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
8472	{
8473	vmxHCReadToTransient<HMVMX_READ_IDT_VECTORING_INFO>(pVCpu, pVmxTransient);
8474	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
8475	{
8476	uint32_t uErrCode;
8477	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
8478	{
8479	vmxHCReadToTransient<HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);
8480	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
8481	}
8482	else
8483	uErrCode = 0;
8484
8485	RTGCUINTPTR GCPtrFaultAddress;
8486	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(pVmxTransient->uIdtVectoringInfo))
8487	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
8488	else
8489	GCPtrFaultAddress = 0;
8490
8491	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
8492
8493	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
8494	pVmxTransient->cbExitInstr, uErrCode, GCPtrFaultAddress);
8495
8496	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo),
8497	VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo)));
8498	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitTaskSwitch);
8499	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8500	}
8501	}
8502
8503	/* Fall back to the interpreter to emulate the task-switch. */
8504	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitTaskSwitch);
8505	return VERR_EM_INTERPRETER;
8506	}
8507
8508
8509	/**
8510	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8511	*/
8512	HMVMX_EXIT_DECL vmxHCExitMtf(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8513	{
8514	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8515
8516	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8517	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
8518	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8519	AssertRC(rc);
8520	return VINF_EM_DBG_STEPPED;
8521	}
8522
8523
8524	/**
8525	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8526	*/
8527	HMVMX_EXIT_DECL vmxHCExitApicAccess(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8528	{
8529	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8530	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitApicAccess);
8531
8532	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
8533	\| HMVMX_READ_EXIT_INSTR_LEN
8534	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
8535	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
8536	\| HMVMX_READ_IDT_VECTORING_INFO
8537	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);
8538
8539	/*
8540	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
8541	*/
8542	VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
8543	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8544	{
8545	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
8546	if (RT_UNLIKELY(VCPU_2_VMXSTATE(pVCpu).Event.fPending))
8547	{
8548	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectInterpret);
8549	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8550	}
8551	}
8552	else
8553	{
8554	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
8555	return rcStrict;
8556	}
8557
8558	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
8559	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8560	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
8561	AssertRCReturn(rc, rc);
8562
8563	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8564	uint32_t const uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
8565	switch (uAccessType)
8566	{
8567	#ifndef IN_NEM_DARWIN
8568	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8569	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8570	{
8571	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8572	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
8573	("vmxHCExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8574
8575	RTGCPHYS GCPhys = VCPU_2_VMXSTATE(pVCpu).vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
8576	GCPhys &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
8577	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
8578	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
8579	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
8580
8581	rcStrict = IOMR0MmioPhysHandler(pVCpu->CTX_SUFF(pVM), pVCpu,
8582	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW, GCPhys);
8583	Log4Func(("IOMR0MmioPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8584	if ( rcStrict == VINF_SUCCESS
8585	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
8586	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
8587	{
8588	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8589	\| HM_CHANGED_GUEST_APIC_TPR);
8590	rcStrict = VINF_SUCCESS;
8591	}
8592	break;
8593	}
8594	#else
8595	/** @todo */
8596	#endif
8597
8598	default:
8599	{
8600	Log4Func(("uAccessType=%#x\n", uAccessType));
8601	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
8602	break;
8603	}
8604	}
8605
8606	if (rcStrict != VINF_SUCCESS)
8607	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchApicAccessToR3);
8608	return rcStrict;
8609	}
8610
8611
8612	/**
8613	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8614	* VM-exit.
8615	*/
8616	HMVMX_EXIT_DECL vmxHCExitMovDRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8617	{
8618	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8619	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8620
8621	/*
8622	* We might also get this VM-exit if the nested-guest isn't intercepting MOV DRx accesses.
8623	* In such a case, rather than disabling MOV DRx intercepts and resuming execution, we
8624	* must emulate the MOV DRx access.
8625	*/
8626	if (!pVmxTransient->fIsNestedGuest)
8627	{
8628	/* We should -not- get this VM-exit if the guest's debug registers were active. */
8629	if (pVmxTransient->fWasGuestDebugStateActive)
8630	{
8631	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
8632	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
8633	}
8634
8635	if ( !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction
8636	&& !pVmxTransient->fWasHyperDebugStateActive)
8637	{
8638	Assert(!DBGFIsStepping(pVCpu));
8639	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
8640
8641	/* Don't intercept MOV DRx any more. */
8642	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
8643	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8644	AssertRC(rc);
8645
8646	#ifndef IN_NEM_DARWIN
8647	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
8648	VMMRZCallRing3Disable(pVCpu);
8649	HM_DISABLE_PREEMPT(pVCpu);
8650
8651	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8652	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
8653	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8654
8655	HM_RESTORE_PREEMPT();
8656	VMMRZCallRing3Enable(pVCpu);
8657	#else
8658	CPUMR3NemActivateGuestDebugState(pVCpu);
8659	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8660	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
8661	#endif
8662
8663	#ifdef VBOX_WITH_STATISTICS
8664	vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
8665	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
8666	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxWrite);
8667	else
8668	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxRead);
8669	#endif
8670	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatDRxContextSwitch);
8671	return VINF_SUCCESS;
8672	}
8673	}
8674
8675	/*
8676	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
8677	* The EFER MSR is always up-to-date.
8678	* Update the segment registers and DR7 from the CPU.
8679	*/
8680	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8681	vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
8682	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
8683	AssertRCReturn(rc, rc);
8684	Log4Func(("cs:rip=%#04x:%08RX64\n", pCtx->cs.Sel, pCtx->rip));
8685
8686	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8687	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
8688	{
8689	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
8690	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
8691	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
8692	if (RT_SUCCESS(rc))
8693	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_DR7);
8694	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxWrite);
8695	}
8696	else
8697	{
8698	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
8699	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
8700	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
8701	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxRead);
8702	}
8703
8704	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8705	if (RT_SUCCESS(rc))
8706	{
8707	int rc2 = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
8708	AssertRCReturn(rc2, rc2);
8709	return VINF_SUCCESS;
8710	}
8711	return rc;
8712	}
8713
8714
8715	/**
8716	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8717	* Conditional VM-exit.
8718	*/
8719	HMVMX_EXIT_DECL vmxHCExitEptMisconfig(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8720	{
8721	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8722
8723	#ifndef IN_NEM_DARWIN
8724	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
8725
8726	vmxHCReadToTransient< HMVMX_READ_EXIT_INSTR_LEN
8727	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
8728	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
8729	\| HMVMX_READ_IDT_VECTORING_INFO
8730	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
8731	\| HMVMX_READ_GUEST_PHYSICAL_ADDR>(pVCpu, pVmxTransient);
8732
8733	/*
8734	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
8735	*/
8736	VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
8737	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8738	{
8739	/*
8740	* In the unlikely case where delivering an event causes an EPT misconfig (MMIO), go back to
8741	* instruction emulation to inject the original event. Otherwise, injecting the original event
8742	* using hardware-assisted VMX would trigger the same EPT misconfig VM-exit again.
8743	*/
8744	if (!VCPU_2_VMXSTATE(pVCpu).Event.fPending)
8745	{ /* likely */ }
8746	else
8747	{
8748	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectInterpret);
8749	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8750	/** @todo NSTVMX: Think about how this should be handled. */
8751	if (pVmxTransient->fIsNestedGuest)
8752	return VERR_VMX_IPE_3;
8753	# endif
8754	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8755	}
8756	}
8757	else
8758	{
8759	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
8760	return rcStrict;
8761	}
8762
8763	/*
8764	* Get sufficient state and update the exit history entry.
8765	*/
8766	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8767	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
8768	AssertRCReturn(rc, rc);
8769
8770	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
8771	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
8772	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
8773	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
8774	if (!pExitRec)
8775	{
8776	/*
8777	* If we succeed, resume guest execution.
8778	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8779	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8780	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8781	* weird case. See @bugref{6043}.
8782	*/
8783	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8784	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8785	/** @todo bird: We can probably just go straight to IOM here and assume that
8786	* it's MMIO, then fall back on PGM if that hunch didn't work out so
8787	* well. However, we need to address that aliasing workarounds that
8788	* PGMR0Trap0eHandlerNPMisconfig implements. So, some care is needed.
8789	*
8790	* Might also be interesting to see if we can get this done more or
8791	* less locklessly inside IOM. Need to consider the lookup table
8792	* updating and use a bit more carefully first (or do all updates via
8793	* rendezvous) */
8794	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
8795	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
8796	if ( rcStrict == VINF_SUCCESS
8797	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
8798	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
8799	{
8800	/* Successfully handled MMIO operation. */
8801	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8802	\| HM_CHANGED_GUEST_APIC_TPR);
8803	rcStrict = VINF_SUCCESS;
8804	}
8805	}
8806	else
8807	{
8808	/*
8809	* Frequent exit or something needing probing. Call EMHistoryExec.
8810	*/
8811	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
8812	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
8813
8814	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
8815	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
8816
8817	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
8818	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
8819	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
8820	}
8821	return rcStrict;
8822	#else
8823	AssertFailed();
8824	return VERR_VMX_IPE_3; /* Should never happen with Apple HV in R3. */
8825	#endif
8826	}
8827
8828
8829	/**
8830	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8831	* VM-exit.
8832	*/
8833	HMVMX_EXIT_DECL vmxHCExitEptViolation(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8834	{
8835	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8836	#ifndef IN_NEM_DARWIN
8837	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
8838
8839	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
8840	\| HMVMX_READ_EXIT_INSTR_LEN
8841	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
8842	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
8843	\| HMVMX_READ_IDT_VECTORING_INFO
8844	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
8845	\| HMVMX_READ_GUEST_PHYSICAL_ADDR>(pVCpu, pVmxTransient);
8846
8847	/*
8848	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
8849	*/
8850	VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
8851	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8852	{
8853	/*
8854	* If delivery of an event causes an EPT violation (true nested #PF and not MMIO),
8855	* we shall resolve the nested #PF and re-inject the original event.
8856	*/
8857	if (VCPU_2_VMXSTATE(pVCpu).Event.fPending)
8858	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectReflectNPF);
8859	}
8860	else
8861	{
8862	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
8863	return rcStrict;
8864	}
8865
8866	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8867	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
8868	AssertRCReturn(rc, rc);
8869
8870	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
8871	uint64_t const uExitQual = pVmxTransient->uExitQual;
8872	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", uExitQual));
8873
8874	RTGCUINT uErrorCode = 0;
8875	if (uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_INSTR_FETCH)
8876	uErrorCode \|= X86_TRAP_PF_ID;
8877	if (uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE)
8878	uErrorCode \|= X86_TRAP_PF_RW;
8879	if (uExitQual & (VMX_EXIT_QUAL_EPT_ENTRY_READ \| VMX_EXIT_QUAL_EPT_ENTRY_WRITE \| VMX_EXIT_QUAL_EPT_ENTRY_EXECUTE))
8880	uErrorCode \|= X86_TRAP_PF_P;
8881
8882	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8883	Log4Func(("at %#RX64 (%#RX64 errcode=%#x) cs:rip=%#04x:%08RX64\n", GCPhys, uExitQual, uErrorCode, pCtx->cs.Sel, pCtx->rip));
8884
8885	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8886
8887	/*
8888	* Handle the pagefault trap for the nested shadow table.
8889	*/
8890	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8891	rcStrict = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
8892	TRPMResetTrap(pVCpu);
8893
8894	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8895	if ( rcStrict == VINF_SUCCESS
8896	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
8897	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
8898	{
8899	/* Successfully synced our nested page tables. */
8900	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitReasonNpf);
8901	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
8902	return VINF_SUCCESS;
8903	}
8904	Log4Func(("EPT return to ring-3 rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8905	return rcStrict;
8906
8907	#else /* IN_NEM_DARWIN */
8908	PVM pVM = pVCpu->CTX_SUFF(pVM);
8909	uint64_t const uHostTsc = ASMReadTSC(); RT_NOREF(uHostTsc);
8910	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
8911	\| HMVMX_READ_GUEST_PHYSICAL_ADDR>(pVCpu, pVmxTransient);
8912	vmxHCImportGuestRip(pVCpu);
8913	vmxHCImportGuestSegReg<X86_SREG_CS>(pVCpu;
8914
8915	/*
8916	* Ask PGM for information about the given GCPhys. We need to check if we're
8917	* out of sync first.
8918	*/
8919	NEMHCDARWINHMACPCCSTATE State = { RT_BOOL(pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE), false, false };
8920	PGMPHYSNEMPAGEINFO Info;
8921	int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, pVmxTransient->uGuestPhysicalAddr, State.fWriteAccess, &Info,
8922	nemR3DarwinHandleMemoryAccessPageCheckerCallback, &State);
8923	if (RT_SUCCESS(rc))
8924	{
8925	if (Info.fNemProt & ( RT_BOOL(pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE)
8926	? NEM_PAGE_PROT_WRITE : NEM_PAGE_PROT_READ))
8927	{
8928	if (State.fCanResume)
8929	{
8930	Log4(("MemExit/%u: %04x:%08RX64: %RGp (=>%RHp) %s fProt=%u%s%s%s; restarting\n",
8931	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
8932	pVmxTransient->uGuestPhysicalAddr, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
8933	Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
8934	State.fDidSomething ? "" : " no-change"));
8935	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_MEMORY_ACCESS),
8936	pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, uHostTsc);
8937	return VINF_SUCCESS;
8938	}
8939	}
8940
8941	Log4(("MemExit/%u: %04x:%08RX64: %RGp (=>%RHp) %s fProt=%u%s%s%s; emulating\n",
8942	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
8943	pVmxTransient->uGuestPhysicalAddr, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
8944	Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
8945	State.fDidSomething ? "" : " no-change"));
8946	}
8947	else
8948	Log4(("MemExit/%u: %04x:%08RX64: %RGp rc=%Rrc%s; emulating\n",
8949	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
8950	pVmxTransient->uGuestPhysicalAddr, rc, State.fDidSomething ? " modified-backing" : ""));
8951
8952	/*
8953	* Emulate the memory access, either access handler or special memory.
8954	*/
8955	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
8956	RT_BOOL(pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE)
8957	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
8958	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
8959	pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, uHostTsc);
8960
8961	rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8962	AssertRCReturn(rc, rc);
8963
8964	VBOXSTRICTRC rcStrict;
8965	if (!pExitRec)
8966	rcStrict = IEMExecOne(pVCpu);
8967	else
8968	{
8969	/* Frequent access or probing. */
8970	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
8971	Log4(("MemExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
8972	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
8973	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
8974	}
8975
8976	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
8977
8978	Log4Func(("EPT return rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8979	return rcStrict;
8980	#endif /* IN_NEM_DARWIN */
8981	}
8982
8983	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8984
8985	/**
8986	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
8987	*/
8988	HMVMX_EXIT_DECL vmxHCExitVmclear(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8989	{
8990	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8991
8992	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
8993	\| HMVMX_READ_EXIT_INSTR_INFO
8994	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
8995	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
8996	\| CPUMCTX_EXTRN_HWVIRT
8997	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
8998	AssertRCReturn(rc, rc);
8999
9000	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
9001
9002	VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
9003	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
9004
9005	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
9006	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9007	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
9008	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9009	{
9010	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9011	rcStrict = VINF_SUCCESS;
9012	}
9013	return rcStrict;
9014	}
9015
9016
9017	/**
9018	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
9019	*/
9020	HMVMX_EXIT_DECL vmxHCExitVmlaunch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9021	{
9022	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9023
9024	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
9025	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
9026	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9027	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
9028	AssertRCReturn(rc, rc);
9029
9030	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
9031
9032	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
9033	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMLAUNCH);
9034	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
9035	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9036	{
9037	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
9038	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
9039	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
9040	}
9041	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
9042	return rcStrict;
9043	}
9044
9045
9046	/**
9047	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
9048	*/
9049	HMVMX_EXIT_DECL vmxHCExitVmptrld(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9050	{
9051	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9052
9053	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
9054	\| HMVMX_READ_EXIT_INSTR_INFO
9055	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9056	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
9057	\| CPUMCTX_EXTRN_HWVIRT
9058	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
9059	AssertRCReturn(rc, rc);
9060
9061	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
9062
9063	VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
9064	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
9065
9066	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
9067	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9068	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
9069	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9070	{
9071	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9072	rcStrict = VINF_SUCCESS;
9073	}
9074	return rcStrict;
9075	}
9076
9077
9078	/**
9079	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
9080	*/
9081	HMVMX_EXIT_DECL vmxHCExitVmptrst(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9082	{
9083	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9084
9085	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
9086	\| HMVMX_READ_EXIT_INSTR_INFO
9087	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9088	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
9089	\| CPUMCTX_EXTRN_HWVIRT
9090	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
9091	AssertRCReturn(rc, rc);
9092
9093	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
9094
9095	VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
9096	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
9097
9098	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
9099	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9100	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
9101	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9102	{
9103	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9104	rcStrict = VINF_SUCCESS;
9105	}
9106	return rcStrict;
9107	}
9108
9109
9110	/**
9111	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
9112	*/
9113	HMVMX_EXIT_DECL vmxHCExitVmread(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9114	{
9115	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9116
9117	/*
9118	* Strictly speaking we should not get VMREAD VM-exits for shadow VMCS fields and
9119	* thus might not need to import the shadow VMCS state, it's safer just in case
9120	* code elsewhere dares look at unsynced VMCS fields.
9121	*/
9122	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
9123	\| HMVMX_READ_EXIT_INSTR_INFO
9124	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9125	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
9126	\| CPUMCTX_EXTRN_HWVIRT
9127	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
9128	AssertRCReturn(rc, rc);
9129
9130	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
9131
9132	VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
9133	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
9134	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
9135
9136	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
9137	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9138	{
9139	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
9140
9141	# if 0 //ndef IN_NEM_DARWIN /** @todo this needs serious tuning still, slows down things enormously. */
9142	/* Try for exit optimization. This is on the following instruction
9143	because it would be a waste of time to have to reinterpret the
9144	already decoded vmwrite instruction. */
9145	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndType(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_VMREAD));
9146	if (pExitRec)
9147	{
9148	/* Frequent access or probing. */
9149	rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
9150	AssertRCReturn(rc, rc);
9151
9152	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
9153	Log4(("vmread/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
9154	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
9155	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
9156	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
9157	}
9158	# endif
9159	}
9160	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9161	{
9162	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9163	rcStrict = VINF_SUCCESS;
9164	}
9165	return rcStrict;
9166	}
9167
9168
9169	/**
9170	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
9171	*/
9172	HMVMX_EXIT_DECL vmxHCExitVmresume(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9173	{
9174	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9175
9176	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
9177	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
9178	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9179	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
9180	AssertRCReturn(rc, rc);
9181
9182	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
9183
9184	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
9185	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMRESUME);
9186	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
9187	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9188	{
9189	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
9190	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
9191	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
9192	}
9193	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
9194	return rcStrict;
9195	}
9196
9197
9198	/**
9199	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
9200	*/
9201	HMVMX_EXIT_DECL vmxHCExitVmwrite(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9202	{
9203	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9204
9205	/*
9206	* Although we should not get VMWRITE VM-exits for shadow VMCS fields, since our HM hook
9207	* gets invoked when IEM's VMWRITE instruction emulation modifies the current VMCS and it
9208	* flags re-loading the entire shadow VMCS, we should save the entire shadow VMCS here.
9209	*/
9210	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
9211	\| HMVMX_READ_EXIT_INSTR_INFO
9212	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9213	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
9214	\| CPUMCTX_EXTRN_HWVIRT
9215	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
9216	AssertRCReturn(rc, rc);
9217
9218	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
9219
9220	VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
9221	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
9222	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
9223
9224	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
9225	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9226	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
9227	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9228	{
9229	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9230	rcStrict = VINF_SUCCESS;
9231	}
9232	return rcStrict;
9233	}
9234
9235
9236	/**
9237	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
9238	*/
9239	HMVMX_EXIT_DECL vmxHCExitVmxoff(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9240	{
9241	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9242
9243	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9244	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
9245	\| CPUMCTX_EXTRN_HWVIRT
9246	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
9247	AssertRCReturn(rc, rc);
9248
9249	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
9250
9251	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbExitInstr);
9252	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9253	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
9254	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9255	{
9256	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9257	rcStrict = VINF_SUCCESS;
9258	}
9259	return rcStrict;
9260	}
9261
9262
9263	/**
9264	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
9265	*/
9266	HMVMX_EXIT_DECL vmxHCExitVmxon(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9267	{
9268	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9269
9270	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
9271	\| HMVMX_READ_EXIT_INSTR_INFO
9272	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9273	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
9274	\| CPUMCTX_EXTRN_HWVIRT
9275	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
9276	AssertRCReturn(rc, rc);
9277
9278	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
9279
9280	VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
9281	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
9282
9283	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
9284	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9285	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
9286	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9287	{
9288	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9289	rcStrict = VINF_SUCCESS;
9290	}
9291	return rcStrict;
9292	}
9293
9294
9295	/**
9296	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
9297	*/
9298	HMVMX_EXIT_DECL vmxHCExitInvvpid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9299	{
9300	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9301
9302	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
9303	\| HMVMX_READ_EXIT_INSTR_INFO
9304	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9305	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
9306	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
9307	AssertRCReturn(rc, rc);
9308
9309	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
9310
9311	VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
9312	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
9313
9314	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
9315	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9316	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
9317	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9318	{
9319	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9320	rcStrict = VINF_SUCCESS;
9321	}
9322	return rcStrict;
9323	}
9324
9325
9326	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
9327	/**
9328	* VM-exit handler for INVEPT (VMX_EXIT_INVEPT). Unconditional VM-exit.
9329	*/
9330	HMVMX_EXIT_DECL vmxHCExitInvept(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9331	{
9332	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9333
9334	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
9335	\| HMVMX_READ_EXIT_INSTR_INFO
9336	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9337	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
9338	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
9339	AssertRCReturn(rc, rc);
9340
9341	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
9342
9343	VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
9344	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
9345
9346	VBOXSTRICTRC rcStrict = IEMExecDecodedInvept(pVCpu, &ExitInfo);
9347	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9348	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
9349	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9350	{
9351	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9352	rcStrict = VINF_SUCCESS;
9353	}
9354	return rcStrict;
9355	}
9356	# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
9357	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
9358	/** @} */
9359
9360
9361	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9362	/** @name Nested-guest VM-exit handlers.
9363	* @{
9364	*/
9365	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9366	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9367	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9368
9369	/**
9370	* Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9371	* Conditional VM-exit.
9372	*/
9373	HMVMX_EXIT_DECL vmxHCExitXcptOrNmiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9374	{
9375	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9376
9377	vmxHCReadToTransient<HMVMX_READ_EXIT_INTERRUPTION_INFO>(pVCpu, pVmxTransient);
9378
9379	uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
9380	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
9381	Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));
9382
9383	switch (uExitIntType)
9384	{
9385	# ifndef IN_NEM_DARWIN
9386	/*
9387	* Physical NMIs:
9388	* We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the host.
9389	*/
9390	case VMX_EXIT_INT_INFO_TYPE_NMI:
9391	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
9392	# endif
9393
9394	/*
9395	* Hardware exceptions,
9396	* Software exceptions,
9397	* Privileged software exceptions:
9398	* Figure out if the exception must be delivered to the guest or the nested-guest.
9399	*/
9400	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
9401	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
9402	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
9403	{
9404	vmxHCReadToTransient< HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
9405	\| HMVMX_READ_EXIT_INSTR_LEN
9406	\| HMVMX_READ_IDT_VECTORING_INFO
9407	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);
9408
9409	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9410	if (CPUMIsGuestVmxXcptInterceptSet(pCtx, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo), pVmxTransient->uExitIntErrorCode))
9411	{
9412	/* Exit qualification is required for debug and page-fault exceptions. */
9413	vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
9414
9415	/*
9416	* For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
9417	* software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
9418	* length. However, if delivery of a software interrupt, software exception or privileged
9419	* software exception causes a VM-exit, that too provides the VM-exit instruction length.
9420	*/
9421	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
9422	VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT(pVmxTransient->uExitIntInfo,
9423	pVmxTransient->uExitIntErrorCode,
9424	pVmxTransient->uIdtVectoringInfo,
9425	pVmxTransient->uIdtVectoringErrorCode);
9426	#ifdef DEBUG_ramshankar
9427	vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
9428	Log4Func(("exit_int_info=%#RX32 err_code=%#RX32 exit_qual=%#RX64\n",
9429	pVmxTransient->uExitIntInfo, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual));
9430	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
9431	Log4Func(("idt_info=%#RX32 idt_errcode=%#RX32 cr2=%#RX64\n",
9432	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uIdtVectoringErrorCode, pCtx->cr2));
9433	#endif
9434	return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
9435	}
9436
9437	/* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs in vmxHCExitXcptPF. */
9438	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
9439	return vmxHCExitXcpt(pVCpu, pVmxTransient);
9440	}
9441
9442	/*
9443	* Software interrupts:
9444	* VM-exits cannot be caused by software interrupts.
9445	*
9446	* External interrupts:
9447	* This should only happen when "acknowledge external interrupts on VM-exit"
9448	* control is set. However, we never set this when executing a guest or
9449	* nested-guest. For nested-guests it is emulated while injecting interrupts into
9450	* the guest.
9451	*/
9452	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
9453	case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
9454	default:
9455	{
9456	VCPU_2_VMXSTATE(pVCpu).u32HMError = pVmxTransient->uExitIntInfo;
9457	return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9458	}
9459	}
9460	}
9461
9462
9463	/**
9464	* Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
9465	* Unconditional VM-exit.
9466	*/
9467	HMVMX_EXIT_DECL vmxHCExitTripleFaultNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9468	{
9469	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9470	return IEMExecVmxVmexitTripleFault(pVCpu);
9471	}
9472
9473
9474	/**
9475	* Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9476	*/
9477	HMVMX_EXIT_NSRC_DECL vmxHCExitIntWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9478	{
9479	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9480
9481	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
9482	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
9483	return vmxHCExitIntWindow(pVCpu, pVmxTransient);
9484	}
9485
9486
9487	/**
9488	* Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9489	*/
9490	HMVMX_EXIT_NSRC_DECL vmxHCExitNmiWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9491	{
9492	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9493
9494	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
9495	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
9496	return vmxHCExitIntWindow(pVCpu, pVmxTransient);
9497	}
9498
9499
9500	/**
9501	* Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
9502	* Unconditional VM-exit.
9503	*/
9504	HMVMX_EXIT_DECL vmxHCExitTaskSwitchNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9505	{
9506	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9507
9508	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
9509	\| HMVMX_READ_EXIT_INSTR_LEN
9510	\| HMVMX_READ_IDT_VECTORING_INFO
9511	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);
9512
9513	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
9514	VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT_ONLY_IDT(pVmxTransient->uIdtVectoringInfo,
9515	pVmxTransient->uIdtVectoringErrorCode);
9516	return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
9517	}
9518
9519
9520	/**
9521	* Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9522	*/
9523	HMVMX_EXIT_DECL vmxHCExitHltNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9524	{
9525	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9526
9527	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
9528	{
9529	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9530	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
9531	}
9532	return vmxHCExitHlt(pVCpu, pVmxTransient);
9533	}
9534
9535
9536	/**
9537	* Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9538	*/
9539	HMVMX_EXIT_DECL vmxHCExitInvlpgNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9540	{
9541	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9542
9543	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
9544	{
9545	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
9546	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9547	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
9548	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
9549	}
9550	return vmxHCExitInvlpg(pVCpu, pVmxTransient);
9551	}
9552
9553
9554	/**
9555	* Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9556	*/
9557	HMVMX_EXIT_DECL vmxHCExitRdpmcNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9558	{
9559	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9560
9561	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
9562	{
9563	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9564	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
9565	}
9566	return vmxHCExitRdpmc(pVCpu, pVmxTransient);
9567	}
9568
9569
9570	/**
9571	* Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
9572	* (VMX_EXIT_VMWRITE). Conditional VM-exit.
9573	*/
9574	HMVMX_EXIT_DECL vmxHCExitVmreadVmwriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9575	{
9576	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9577
9578	Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD
9579	\|\| pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
9580
9581	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_INFO>(pVCpu, pVmxTransient);
9582
9583	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
9584	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
9585	uint64_t u64VmcsField = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
9586
9587	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
9588	if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
9589	u64VmcsField &= UINT64_C(0xffffffff);
9590
9591	if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64VmcsField))
9592	{
9593	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
9594	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9595	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
9596	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
9597	}
9598
9599	if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
9600	return vmxHCExitVmread(pVCpu, pVmxTransient);
9601	return vmxHCExitVmwrite(pVCpu, pVmxTransient);
9602	}
9603
9604
9605	/**
9606	* Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9607	*/
9608	HMVMX_EXIT_DECL vmxHCExitRdtscNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9609	{
9610	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9611
9612	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
9613	{
9614	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9615	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
9616	}
9617
9618	return vmxHCExitRdtsc(pVCpu, pVmxTransient);
9619	}
9620
9621
9622	/**
9623	* Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
9624	* Conditional VM-exit.
9625	*/
9626	HMVMX_EXIT_DECL vmxHCExitMovCRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9627	{
9628	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9629
9630	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
9631	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9632
9633	VBOXSTRICTRC rcStrict;
9634	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
9635	switch (uAccessType)
9636	{
9637	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
9638	{
9639	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
9640	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
9641	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
9642	uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
9643
9644	bool fIntercept;
9645	switch (iCrReg)
9646	{
9647	case 0:
9648	case 4:
9649	fIntercept = CPUMIsGuestVmxMovToCr0Cr4InterceptSet(&pVCpu->cpum.GstCtx, iCrReg, uNewCrX);
9650	break;
9651
9652	case 3:
9653	fIntercept = CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCrX);
9654	break;
9655
9656	case 8:
9657	fIntercept = CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_CR8_LOAD_EXIT);
9658	break;
9659
9660	default:
9661	fIntercept = false;
9662	break;
9663	}
9664	if (fIntercept)
9665	{
9666	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
9667	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
9668	}
9669	else
9670	{
9671	int const rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
9672	AssertRCReturn(rc, rc);
9673	rcStrict = vmxHCExitMovToCrX(pVCpu, pVmxTransient->cbExitInstr, iGReg, iCrReg);
9674	}
9675	break;
9676	}
9677
9678	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
9679	{
9680	/*
9681	* CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
9682	* CR2 reads do not cause a VM-exit.
9683	* CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
9684	* CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
9685	*/
9686	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
9687	if ( iCrReg == 3
9688	\|\| iCrReg == 8)
9689	{
9690	static const uint32_t s_auCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
9691	0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
9692	uint32_t const uIntercept = s_auCrXReadIntercepts[iCrReg];
9693	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, uIntercept))
9694	{
9695	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
9696	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
9697	}
9698	else
9699	{
9700	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
9701	rcStrict = vmxHCExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
9702	}
9703	}
9704	else
9705	{
9706	AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
9707	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
9708	}
9709	break;
9710	}
9711
9712	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
9713	{
9714	PCVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
9715	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
9716	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
9717	if ( (uGstHostMask & X86_CR0_TS)
9718	&& (uReadShadow & X86_CR0_TS))
9719	{
9720	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
9721	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
9722	}
9723	else
9724	rcStrict = vmxHCExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr);
9725	break;
9726	}
9727
9728	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
9729	{
9730	RTGCPTR GCPtrEffDst;
9731	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
9732	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
9733	if (fMemOperand)
9734	{
9735	vmxHCReadToTransient<HMVMX_READ_GUEST_LINEAR_ADDR>(pVCpu, pVmxTransient);
9736	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
9737	}
9738	else
9739	GCPtrEffDst = NIL_RTGCPTR;
9740
9741	if (CPUMIsGuestVmxLmswInterceptSet(&pVCpu->cpum.GstCtx, uNewMsw))
9742	{
9743	VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
9744	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
9745	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
9746	}
9747	else
9748	rcStrict = vmxHCExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, uNewMsw, GCPtrEffDst);
9749	break;
9750	}
9751
9752	default:
9753	{
9754	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
9755	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
9756	}
9757	}
9758
9759	if (rcStrict == VINF_IEM_RAISED_XCPT)
9760	{
9761	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9762	rcStrict = VINF_SUCCESS;
9763	}
9764	return rcStrict;
9765	}
9766
9767
9768	/**
9769	* Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
9770	* Conditional VM-exit.
9771	*/
9772	HMVMX_EXIT_DECL vmxHCExitMovDRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9773	{
9774	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9775
9776	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
9777	{
9778	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
9779	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9780	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
9781	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
9782	}
9783	return vmxHCExitMovDRx(pVCpu, pVmxTransient);
9784	}
9785
9786
9787	/**
9788	* Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
9789	* Conditional VM-exit.
9790	*/
9791	HMVMX_EXIT_DECL vmxHCExitIoInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9792	{
9793	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9794
9795	vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
9796
9797	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
9798	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
9799	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
9800
9801	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
9802	uint8_t const cbAccess = s_aIOSizes[uIOSize];
9803	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
9804	{
9805	/*
9806	* IN/OUT instruction:
9807	* - Provides VM-exit instruction length.
9808	*
9809	* INS/OUTS instruction:
9810	* - Provides VM-exit instruction length.
9811	* - Provides Guest-linear address.
9812	* - Optionally provides VM-exit instruction info (depends on CPU feature).
9813	*/
9814	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
9815	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9816
9817	/* Make sure we don't use stale/uninitialized VMX-transient info. below. */
9818	pVmxTransient->ExitInstrInfo.u = 0;
9819	pVmxTransient->uGuestLinearAddr = 0;
9820
9821	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
9822	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
9823	if (fIOString)
9824	{
9825	vmxHCReadToTransient<HMVMX_READ_GUEST_LINEAR_ADDR>(pVCpu, pVmxTransient);
9826	if (fVmxInsOutsInfo)
9827	{
9828	Assert(RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
9829	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_INFO>(pVCpu, pVmxTransient);
9830	}
9831	}
9832
9833	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_AND_LIN_ADDR_FROM_TRANSIENT(pVmxTransient);
9834	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
9835	}
9836	return vmxHCExitIoInstr(pVCpu, pVmxTransient);
9837	}
9838
9839
9840	/**
9841	* Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9842	*/
9843	HMVMX_EXIT_DECL vmxHCExitRdmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9844	{
9845	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9846
9847	uint32_t fMsrpm;
9848	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
9849	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap, pVCpu->cpum.GstCtx.ecx);
9850	else
9851	fMsrpm = VMXMSRPM_EXIT_RD;
9852
9853	if (fMsrpm & VMXMSRPM_EXIT_RD)
9854	{
9855	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9856	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
9857	}
9858	return vmxHCExitRdmsr(pVCpu, pVmxTransient);
9859	}
9860
9861
9862	/**
9863	* Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9864	*/
9865	HMVMX_EXIT_DECL vmxHCExitWrmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9866	{
9867	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9868
9869	uint32_t fMsrpm;
9870	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
9871	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap, pVCpu->cpum.GstCtx.ecx);
9872	else
9873	fMsrpm = VMXMSRPM_EXIT_WR;
9874
9875	if (fMsrpm & VMXMSRPM_EXIT_WR)
9876	{
9877	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9878	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
9879	}
9880	return vmxHCExitWrmsr(pVCpu, pVmxTransient);
9881	}
9882
9883
9884	/**
9885	* Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9886	*/
9887	HMVMX_EXIT_DECL vmxHCExitMwaitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9888	{
9889	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9890
9891	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
9892	{
9893	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9894	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
9895	}
9896	return vmxHCExitMwait(pVCpu, pVmxTransient);
9897	}
9898
9899
9900	/**
9901	* Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
9902	* VM-exit.
9903	*/
9904	HMVMX_EXIT_DECL vmxHCExitMtfNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9905	{
9906	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9907
9908	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
9909	vmxHCReadToTransient<HMVMX_READ_GUEST_PENDING_DBG_XCPTS>(pVCpu, pVmxTransient);
9910	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_DBG_XCPTS_FROM_TRANSIENT(pVmxTransient);
9911	return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
9912	}
9913
9914
9915	/**
9916	* Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9917	*/
9918	HMVMX_EXIT_DECL vmxHCExitMonitorNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9919	{
9920	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9921
9922	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
9923	{
9924	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9925	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
9926	}
9927	return vmxHCExitMonitor(pVCpu, pVmxTransient);
9928	}
9929
9930
9931	/**
9932	* Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
9933	*/
9934	HMVMX_EXIT_DECL vmxHCExitPauseNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9935	{
9936	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9937
9938	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
9939	* PAUSE when executing a nested-guest? If it does not, we would not need
9940	* to check for the intercepts here. Just call VM-exit... */
9941
9942	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
9943	if ( CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
9944	\|\| CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
9945	{
9946	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9947	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
9948	}
9949	return vmxHCExitPause(pVCpu, pVmxTransient);
9950	}
9951
9952
9953	/**
9954	* Nested-guest VM-exit handler for when the TPR value is lowered below the
9955	* specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9956	*/
9957	HMVMX_EXIT_NSRC_DECL vmxHCExitTprBelowThresholdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9958	{
9959	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9960
9961	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
9962	{
9963	vmxHCReadToTransient<HMVMX_READ_GUEST_PENDING_DBG_XCPTS>(pVCpu, pVmxTransient);
9964	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_DBG_XCPTS_FROM_TRANSIENT(pVmxTransient);
9965	return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
9966	}
9967	return vmxHCExitTprBelowThreshold(pVCpu, pVmxTransient);
9968	}
9969
9970
9971	/**
9972	* Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
9973	* VM-exit.
9974	*/
9975	HMVMX_EXIT_DECL vmxHCExitApicAccessNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9976	{
9977	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9978
9979	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
9980	\| HMVMX_READ_EXIT_INSTR_LEN
9981	\| HMVMX_READ_IDT_VECTORING_INFO
9982	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);
9983
9984	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
9985
9986	Log4Func(("at offset %#x type=%u\n", VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual),
9987	VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual)));
9988
9989	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
9990	VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT_ONLY_IDT(pVmxTransient->uIdtVectoringInfo,
9991	pVmxTransient->uIdtVectoringErrorCode);
9992	return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
9993	}
9994
9995
9996	/**
9997	* Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
9998	* Conditional VM-exit.
9999	*/
10000	HMVMX_EXIT_DECL vmxHCExitApicWriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10001	{
10002	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10003
10004	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
10005	vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
10006	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
10007	}
10008
10009
10010	/**
10011	* Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
10012	* Conditional VM-exit.
10013	*/
10014	HMVMX_EXIT_DECL vmxHCExitVirtEoiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10015	{
10016	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10017
10018	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
10019	vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
10020	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
10021	}
10022
10023
10024	/**
10025	* Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
10026	*/
10027	HMVMX_EXIT_DECL vmxHCExitRdtscpNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10028	{
10029	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10030
10031	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
10032	{
10033	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
10034	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
10035	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
10036	}
10037	return vmxHCExitRdtscp(pVCpu, pVmxTransient);
10038	}
10039
10040
10041	/**
10042	* Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
10043	*/
10044	HMVMX_EXIT_NSRC_DECL vmxHCExitWbinvdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10045	{
10046	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10047
10048	if (CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
10049	{
10050	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
10051	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
10052	}
10053	return vmxHCExitWbinvd(pVCpu, pVmxTransient);
10054	}
10055
10056
10057	/**
10058	* Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10059	*/
10060	HMVMX_EXIT_DECL vmxHCExitInvpcidNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10061	{
10062	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10063
10064	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
10065	{
10066	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
10067	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
10068	\| HMVMX_READ_EXIT_INSTR_INFO
10069	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
10070	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
10071	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
10072	}
10073	return vmxHCExitInvpcid(pVCpu, pVmxTransient);
10074	}
10075
10076
10077	/**
10078	* Nested-guest VM-exit handler for invalid-guest state
10079	* (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
10080	*/
10081	HMVMX_EXIT_DECL vmxHCExitErrInvalidGuestStateNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10082	{
10083	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10084
10085	/*
10086	* Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
10087	* So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
10088	* Handle it like it's in an invalid guest state of the outer guest.
10089	*
10090	* When the fast path is implemented, this should be changed to cause the corresponding
10091	* nested-guest VM-exit.
10092	*/
10093	return vmxHCExitErrInvalidGuestState(pVCpu, pVmxTransient);
10094	}
10095
10096
10097	/**
10098	* Nested-guest VM-exit handler for instructions that cause VM-exits unconditionally
10099	* and only provide the instruction length.
10100	*
10101	* Unconditional VM-exit.
10102	*/
10103	HMVMX_EXIT_DECL vmxHCExitInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10104	{
10105	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10106
10107	#ifdef VBOX_STRICT
10108	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10109	switch (pVmxTransient->uExitReason)
10110	{
10111	case VMX_EXIT_ENCLS:
10112	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
10113	break;
10114
10115	case VMX_EXIT_VMFUNC:
10116	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_VMFUNC));
10117	break;
10118	}
10119	#endif
10120
10121	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
10122	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
10123	}
10124
10125
10126	/**
10127	* Nested-guest VM-exit handler for instructions that provide instruction length as
10128	* well as more information.
10129	*
10130	* Unconditional VM-exit.
10131	*/
10132	HMVMX_EXIT_DECL vmxHCExitInstrWithInfoNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10133	{
10134	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10135
10136	# ifdef VBOX_STRICT
10137	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10138	switch (pVmxTransient->uExitReason)
10139	{
10140	case VMX_EXIT_GDTR_IDTR_ACCESS:
10141	case VMX_EXIT_LDTR_TR_ACCESS:
10142	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
10143	break;
10144
10145	case VMX_EXIT_RDRAND:
10146	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
10147	break;
10148
10149	case VMX_EXIT_RDSEED:
10150	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
10151	break;
10152
10153	case VMX_EXIT_XSAVES:
10154	case VMX_EXIT_XRSTORS:
10155	/** @todo NSTVMX: Verify XSS-bitmap. */
10156	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
10157	break;
10158
10159	case VMX_EXIT_UMWAIT:
10160	case VMX_EXIT_TPAUSE:
10161	Assert(CPUMIsGuestVmxProcCtlsSet(pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
10162	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
10163	break;
10164
10165	case VMX_EXIT_LOADIWKEY:
10166	Assert(CPUMIsGuestVmxProcCtls3Set(pCtx, VMX_PROC_CTLS3_LOADIWKEY_EXIT));
10167	break;
10168	}
10169	# endif
10170
10171	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
10172	\| HMVMX_READ_EXIT_INSTR_LEN
10173	\| HMVMX_READ_EXIT_INSTR_INFO>(pVCpu, pVmxTransient);
10174	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
10175	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
10176	}
10177
10178	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
10179
10180	/**
10181	* Nested-guest VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION).
10182	* Conditional VM-exit.
10183	*/
10184	HMVMX_EXIT_DECL vmxHCExitEptViolationNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10185	{
10186	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10187	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
10188
10189	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10190	if (CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_EPT))
10191	{
10192	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10193	AssertRCReturn(rc, rc);
10194
10195	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
10196	\| HMVMX_READ_EXIT_INSTR_LEN
10197	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
10198	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
10199	\| HMVMX_READ_IDT_VECTORING_INFO
10200	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
10201	\| HMVMX_READ_GUEST_PHYSICAL_ADDR>(pVCpu, pVmxTransient);
10202
10203	/*
10204	* If it's our VMEXIT, we're responsible for re-injecting any event which delivery
10205	* might have triggered this VMEXIT. If we forward the problem to the inner VMM,
10206	* it's its problem to deal with that issue and we'll clear the recovered event.
10207	*/
10208	VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
10209	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10210	{ /likely/ }
10211	else
10212	{
10213	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
10214	return rcStrict;
10215	}
10216	bool const fClearEventOnForward = VCPU_2_VMXSTATE(pVCpu).Event.fPending; /* paranoia. should not inject events below. */
10217
10218	RTGCPHYS const GCPhysNestedFault = pVmxTransient->uGuestPhysicalAddr;
10219	uint64_t const uExitQual = pVmxTransient->uExitQual;
10220
10221	RTGCPTR GCPtrNestedFault;
10222	bool const fIsLinearAddrValid = RT_BOOL(uExitQual & VMX_EXIT_QUAL_EPT_LINEAR_ADDR_VALID);
10223	if (fIsLinearAddrValid)
10224	{
10225	vmxHCReadToTransient<HMVMX_READ_GUEST_LINEAR_ADDR>(pVCpu, pVmxTransient);
10226	GCPtrNestedFault = pVmxTransient->uGuestLinearAddr;
10227	}
10228	else
10229	GCPtrNestedFault = 0;
10230
10231	RTGCUINT const uErr = ((uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_INSTR_FETCH) ? X86_TRAP_PF_ID : 0)
10232	\| ((uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE) ? X86_TRAP_PF_RW : 0)
10233	\| ((uExitQual & ( VMX_EXIT_QUAL_EPT_ENTRY_READ
10234	\| VMX_EXIT_QUAL_EPT_ENTRY_WRITE
10235	\| VMX_EXIT_QUAL_EPT_ENTRY_EXECUTE)) ? X86_TRAP_PF_P : 0);
10236
10237	PGMPTWALK Walk;
10238	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10239	rcStrict = PGMR0NestedTrap0eHandlerNestedPaging(pVCpu, PGMMODE_EPT, uErr, CPUMCTX2CORE(pCtx),
10240	GCPhysNestedFault, fIsLinearAddrValid, GCPtrNestedFault,
10241	&Walk);
10242	Log7Func(("PGM (uExitQual=%#RX64, %RGp, %RGv) -> %Rrc (fFailed=%d)\n",
10243	uExitQual, GCPhysNestedFault, GCPtrNestedFault, VBOXSTRICTRC_VAL(rcStrict), Walk.fFailed));
10244	if (RT_SUCCESS(rcStrict))
10245	return rcStrict;
10246
10247	if (fClearEventOnForward)
10248	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false;
10249
10250	VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT_ONLY_IDT(pVmxTransient->uIdtVectoringInfo,
10251	pVmxTransient->uIdtVectoringErrorCode);
10252	if (Walk.fFailed & PGM_WALKFAIL_EPT_VIOLATION)
10253	{
10254	VMXVEXITINFO const ExitInfo
10255	= VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_AND_GST_ADDRESSES(VMX_EXIT_EPT_VIOLATION,
10256	pVmxTransient->uExitQual,
10257	pVmxTransient->cbExitInstr,
10258	pVmxTransient->uGuestLinearAddr,
10259	pVmxTransient->uGuestPhysicalAddr);
10260	return IEMExecVmxVmexitEptViolation(pVCpu, &ExitInfo, &ExitEventInfo);
10261	}
10262
10263	Assert(Walk.fFailed & PGM_WALKFAIL_EPT_MISCONFIG);
10264	return IEMExecVmxVmexitEptMisconfig(pVCpu, pVmxTransient->uGuestPhysicalAddr, &ExitEventInfo);
10265	}
10266
10267	return vmxHCExitEptViolation(pVCpu, pVmxTransient);
10268	}
10269
10270
10271	/**
10272	* Nested-guest VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10273	* Conditional VM-exit.
10274	*/
10275	HMVMX_EXIT_DECL vmxHCExitEptMisconfigNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10276	{
10277	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10278	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
10279
10280	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10281	if (CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_EPT))
10282	{
10283	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_ALL);
10284	AssertRCReturn(rc, rc);
10285
10286	vmxHCReadToTransient<HMVMX_READ_GUEST_PHYSICAL_ADDR>(pVCpu, pVmxTransient);
10287
10288	PGMPTWALK Walk;
10289	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10290	RTGCPHYS const GCPhysNestedFault = pVmxTransient->uGuestPhysicalAddr;
10291	VBOXSTRICTRC rcStrict = PGMR0NestedTrap0eHandlerNestedPaging(pVCpu, PGMMODE_EPT, X86_TRAP_PF_RSVD, CPUMCTX2CORE(pCtx),
10292	GCPhysNestedFault, false /* fIsLinearAddrValid */,
10293	0 /* GCPtrNestedFault */, &Walk);
10294	if (RT_SUCCESS(rcStrict))
10295	{
10296	AssertMsgFailed(("Shouldn't happen with the way we have programmed the EPT shadow tables\n"));
10297	return rcStrict;
10298	}
10299
10300	AssertMsg(Walk.fFailed & PGM_WALKFAIL_EPT_MISCONFIG, ("GCPhysNestedFault=%#RGp\n", GCPhysNestedFault));
10301	vmxHCReadToTransient< HMVMX_READ_IDT_VECTORING_INFO
10302	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);
10303
10304	VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT_ONLY_IDT(pVmxTransient->uIdtVectoringInfo,
10305	pVmxTransient->uIdtVectoringErrorCode);
10306	return IEMExecVmxVmexitEptMisconfig(pVCpu, pVmxTransient->uGuestPhysicalAddr, &ExitEventInfo);
10307	}
10308
10309	return vmxHCExitEptMisconfig(pVCpu, pVmxTransient);
10310	}
10311
10312	# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
10313
10314	/** @} */
10315	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10316
10317
10318	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
10319	* probes.
10320	*
10321	* The following few functions and associated structure contains the bloat
10322	* necessary for providing detailed debug events and dtrace probes as well as
10323	* reliable host side single stepping. This works on the principle of
10324	* "subclassing" the normal execution loop and workers. We replace the loop
10325	* method completely and override selected helpers to add necessary adjustments
10326	* to their core operation.
10327	*
10328	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
10329	* any performance for debug and analysis features.
10330	*
10331	* @{
10332	*/
10333
10334	/**
10335	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
10336	* the debug run loop.
10337	*/
10338	typedef struct VMXRUNDBGSTATE
10339	{
10340	/** The RIP we started executing at. This is for detecting that we stepped. */
10341	uint64_t uRipStart;
10342	/** The CS we started executing with. */
10343	uint16_t uCsStart;
10344
10345	/** Whether we've actually modified the 1st execution control field. */
10346	bool fModifiedProcCtls : 1;
10347	/** Whether we've actually modified the 2nd execution control field. */
10348	bool fModifiedProcCtls2 : 1;
10349	/** Whether we've actually modified the exception bitmap. */
10350	bool fModifiedXcptBitmap : 1;
10351
10352	/** We desire the modified the CR0 mask to be cleared. */
10353	bool fClearCr0Mask : 1;
10354	/** We desire the modified the CR4 mask to be cleared. */
10355	bool fClearCr4Mask : 1;
10356	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
10357	uint32_t fCpe1Extra;
10358	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
10359	uint32_t fCpe1Unwanted;
10360	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
10361	uint32_t fCpe2Extra;
10362	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
10363	uint32_t bmXcptExtra;
10364	/** The sequence number of the Dtrace provider settings the state was
10365	* configured against. */
10366	uint32_t uDtraceSettingsSeqNo;
10367	/** VM-exits to check (one bit per VM-exit). */
10368	uint32_t bmExitsToCheck[3];
10369
10370	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
10371	uint32_t fProcCtlsInitial;
10372	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
10373	uint32_t fProcCtls2Initial;
10374	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
10375	uint32_t bmXcptInitial;
10376	} VMXRUNDBGSTATE;
10377	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
10378	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
10379
10380
10381	/**
10382	* Initializes the VMXRUNDBGSTATE structure.
10383	*
10384	* @param pVCpu The cross context virtual CPU structure of the
10385	* calling EMT.
10386	* @param pVmxTransient The VMX-transient structure.
10387	* @param pDbgState The debug state to initialize.
10388	*/
10389	static void vmxHCRunDebugStateInit(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
10390	{
10391	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
10392	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
10393
10394	pDbgState->fModifiedProcCtls = false;
10395	pDbgState->fModifiedProcCtls2 = false;
10396	pDbgState->fModifiedXcptBitmap = false;
10397	pDbgState->fClearCr0Mask = false;
10398	pDbgState->fClearCr4Mask = false;
10399	pDbgState->fCpe1Extra = 0;
10400	pDbgState->fCpe1Unwanted = 0;
10401	pDbgState->fCpe2Extra = 0;
10402	pDbgState->bmXcptExtra = 0;
10403	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
10404	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
10405	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
10406	}
10407
10408
10409	/**
10410	* Updates the VMSC fields with changes requested by @a pDbgState.
10411	*
10412	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
10413	* immediately before executing guest code, i.e. when interrupts are disabled.
10414	* We don't check status codes here as we cannot easily assert or return in the
10415	* latter case.
10416	*
10417	* @param pVCpu The cross context virtual CPU structure.
10418	* @param pVmxTransient The VMX-transient structure.
10419	* @param pDbgState The debug state.
10420	*/
10421	static void vmxHCPreRunGuestDebugStateApply(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
10422	{
10423	/*
10424	* Ensure desired flags in VMCS control fields are set.
10425	* (Ignoring write failure here, as we're committed and it's just debug extras.)
10426	*
10427	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
10428	* there should be no stale data in pCtx at this point.
10429	*/
10430	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10431	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
10432	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
10433	{
10434	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
10435	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
10436	VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
10437	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
10438	pDbgState->fModifiedProcCtls = true;
10439	}
10440
10441	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
10442	{
10443	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
10444	VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
10445	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
10446	pDbgState->fModifiedProcCtls2 = true;
10447	}
10448
10449	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
10450	{
10451	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
10452	VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
10453	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
10454	pDbgState->fModifiedXcptBitmap = true;
10455	}
10456
10457	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
10458	{
10459	pVmcsInfo->u64Cr0Mask = 0;
10460	VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_MASK, 0);
10461	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
10462	}
10463
10464	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
10465	{
10466	pVmcsInfo->u64Cr4Mask = 0;
10467	VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR4_MASK, 0);
10468	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
10469	}
10470
10471	NOREF(pVCpu);
10472	}
10473
10474
10475	/**
10476	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
10477	* re-entry next time around.
10478	*
10479	* @returns Strict VBox status code (i.e. informational status codes too).
10480	* @param pVCpu The cross context virtual CPU structure.
10481	* @param pVmxTransient The VMX-transient structure.
10482	* @param pDbgState The debug state.
10483	* @param rcStrict The return code from executing the guest using single
10484	* stepping.
10485	*/
10486	static VBOXSTRICTRC vmxHCRunDebugStateRevert(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
10487	VBOXSTRICTRC rcStrict)
10488	{
10489	/*
10490	* Restore VM-exit control settings as we may not reenter this function the
10491	* next time around.
10492	*/
10493	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10494
10495	/* We reload the initial value, trigger what we can of recalculations the
10496	next time around. From the looks of things, that's all that's required atm. */
10497	if (pDbgState->fModifiedProcCtls)
10498	{
10499	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
10500	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
10501	int rc2 = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
10502	AssertRC(rc2);
10503	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
10504	}
10505
10506	/* We're currently the only ones messing with this one, so just restore the
10507	cached value and reload the field. */
10508	if ( pDbgState->fModifiedProcCtls2
10509	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
10510	{
10511	int rc2 = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
10512	AssertRC(rc2);
10513	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
10514	}
10515
10516	/* If we've modified the exception bitmap, we restore it and trigger
10517	reloading and partial recalculation the next time around. */
10518	if (pDbgState->fModifiedXcptBitmap)
10519	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
10520
10521	return rcStrict;
10522	}
10523
10524
10525	/**
10526	* Configures VM-exit controls for current DBGF and DTrace settings.
10527	*
10528	* This updates @a pDbgState and the VMCS execution control fields to reflect
10529	* the necessary VM-exits demanded by DBGF and DTrace.
10530	*
10531	* @param pVCpu The cross context virtual CPU structure.
10532	* @param pVmxTransient The VMX-transient structure. May update
10533	* fUpdatedTscOffsettingAndPreemptTimer.
10534	* @param pDbgState The debug state.
10535	*/
10536	static void vmxHCPreRunGuestDebugStateUpdate(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
10537	{
10538	#ifndef IN_NEM_DARWIN
10539	/*
10540	* Take down the dtrace serial number so we can spot changes.
10541	*/
10542	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
10543	ASMCompilerBarrier();
10544	#endif
10545
10546	/*
10547	* We'll rebuild most of the middle block of data members (holding the
10548	* current settings) as we go along here, so start by clearing it all.
10549	*/
10550	pDbgState->bmXcptExtra = 0;
10551	pDbgState->fCpe1Extra = 0;
10552	pDbgState->fCpe1Unwanted = 0;
10553	pDbgState->fCpe2Extra = 0;
10554	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
10555	pDbgState->bmExitsToCheck[i] = 0;
10556
10557	/*
10558	* Software interrupts (INT XXh) - no idea how to trigger these...
10559	*/
10560	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10561	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
10562	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
10563	{
10564	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
10565	}
10566
10567	/*
10568	* INT3 breakpoints - triggered by #BP exceptions.
10569	*/
10570	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
10571	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
10572
10573	/*
10574	* Exception bitmap and XCPT events+probes.
10575	*/
10576	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
10577	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
10578	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
10579
10580	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
10581	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
10582	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
10583	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
10584	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
10585	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
10586	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
10587	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
10588	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
10589	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
10590	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
10591	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
10592	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
10593	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
10594	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
10595	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
10596	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
10597	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
10598
10599	if (pDbgState->bmXcptExtra)
10600	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
10601
10602	/*
10603	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
10604	*
10605	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
10606	* So, when adding/changing/removing please don't forget to update it.
10607	*
10608	* Some of the macros are picking up local variables to save horizontal space,
10609	* (being able to see it in a table is the lesser evil here).
10610	*/
10611	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
10612	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
10613	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
10614	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
10615	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
10616	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
10617	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
10618	} else do { } while (0)
10619	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
10620	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
10621	{ \
10622	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
10623	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
10624	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
10625	} else do { } while (0)
10626	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
10627	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
10628	{ \
10629	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
10630	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
10631	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
10632	} else do { } while (0)
10633	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
10634	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
10635	{ \
10636	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
10637	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
10638	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
10639	} else do { } while (0)
10640
10641	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
10642	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
10643	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
10644	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
10645	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
10646
10647	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
10648	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
10649	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
10650	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
10651	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
10652	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
10653	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
10654	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
10655	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
10656	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
10657	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
10658	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
10659	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
10660	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
10661	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
10662	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
10663	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
10664	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
10665	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
10666	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
10667	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
10668	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
10669	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
10670	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
10671	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
10672	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
10673	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
10674	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
10675	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
10676	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
10677	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
10678	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
10679	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
10680	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
10681	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
10682	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
10683
10684	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
10685	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
10686	{
10687	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
10688	\| CPUMCTX_EXTRN_APIC_TPR);
10689	AssertRC(rc);
10690
10691	#if 0 /** @todo fix me */
10692	pDbgState->fClearCr0Mask = true;
10693	pDbgState->fClearCr4Mask = true;
10694	#endif
10695	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
10696	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
10697	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
10698	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10699	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
10700	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
10701	require clearing here and in the loop if we start using it. */
10702	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
10703	}
10704	else
10705	{
10706	if (pDbgState->fClearCr0Mask)
10707	{
10708	pDbgState->fClearCr0Mask = false;
10709	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_CR0);
10710	}
10711	if (pDbgState->fClearCr4Mask)
10712	{
10713	pDbgState->fClearCr4Mask = false;
10714	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_CR4);
10715	}
10716	}
10717	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
10718	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
10719
10720	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
10721	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
10722	{
10723	/** @todo later, need to fix handler as it assumes this won't usually happen. */
10724	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
10725	}
10726	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
10727	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
10728
10729	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
10730	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
10731	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
10732	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
10733	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
10734	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
10735	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
10736	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
10737	#if 0 /** @todo too slow, fix handler. */
10738	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
10739	#endif
10740	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
10741
10742	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
10743	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
10744	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
10745	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
10746	{
10747	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
10748	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
10749	}
10750	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
10751	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
10752	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
10753	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
10754
10755	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
10756	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
10757	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
10758	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
10759	{
10760	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
10761	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
10762	}
10763	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
10764	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
10765	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
10766	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
10767
10768	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
10769	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
10770	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
10771	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
10772	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
10773	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
10774	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
10775	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
10776	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
10777	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
10778	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
10779	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
10780	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
10781	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
10782	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
10783	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
10784	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
10785	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
10786	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
10787	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
10788	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
10789	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
10790
10791	#undef IS_EITHER_ENABLED
10792	#undef SET_ONLY_XBM_IF_EITHER_EN
10793	#undef SET_CPE1_XBM_IF_EITHER_EN
10794	#undef SET_CPEU_XBM_IF_EITHER_EN
10795	#undef SET_CPE2_XBM_IF_EITHER_EN
10796
10797	/*
10798	* Sanitize the control stuff.
10799	*/
10800	pDbgState->fCpe2Extra &= g_HmMsrs.u.vmx.ProcCtls2.n.allowed1;
10801	if (pDbgState->fCpe2Extra)
10802	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
10803	pDbgState->fCpe1Extra &= g_HmMsrs.u.vmx.ProcCtls.n.allowed1;
10804	pDbgState->fCpe1Unwanted &= ~g_HmMsrs.u.vmx.ProcCtls.n.allowed0;
10805	#ifndef IN_NEM_DARWIN
10806	if (pVCpu->hmr0.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
10807	{
10808	pVCpu->hmr0.s.fDebugWantRdTscExit ^= true;
10809	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
10810	}
10811	#else
10812	if (pVCpu->nem.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
10813	{
10814	pVCpu->nem.s.fDebugWantRdTscExit ^= true;
10815	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
10816	}
10817	#endif
10818
10819	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
10820	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
10821	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
10822	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
10823	}
10824
10825
10826	/**
10827	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
10828	* appropriate.
10829	*
10830	* The caller has checked the VM-exit against the
10831	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
10832	* already, so we don't have to do that either.
10833	*
10834	* @returns Strict VBox status code (i.e. informational status codes too).
10835	* @param pVCpu The cross context virtual CPU structure.
10836	* @param pVmxTransient The VMX-transient structure.
10837	* @param uExitReason The VM-exit reason.
10838	*
10839	* @remarks The name of this function is displayed by dtrace, so keep it short
10840	* and to the point. No longer than 33 chars long, please.
10841	*/
10842	static VBOXSTRICTRC vmxHCHandleExitDtraceEvents(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
10843	{
10844	/*
10845	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
10846	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
10847	*
10848	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
10849	* does. Must add/change/remove both places. Same ordering, please.
10850	*
10851	* Added/removed events must also be reflected in the next section
10852	* where we dispatch dtrace events.
10853	*/
10854	bool fDtrace1 = false;
10855	bool fDtrace2 = false;
10856	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
10857	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
10858	uint32_t uEventArg = 0;
10859	#define SET_EXIT(a_EventSubName) \
10860	do { \
10861	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
10862	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
10863	} while (0)
10864	#define SET_BOTH(a_EventSubName) \
10865	do { \
10866	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
10867	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
10868	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
10869	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
10870	} while (0)
10871	switch (uExitReason)
10872	{
10873	case VMX_EXIT_MTF:
10874	return vmxHCExitMtf(pVCpu, pVmxTransient);
10875
10876	case VMX_EXIT_XCPT_OR_NMI:
10877	{
10878	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
10879	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
10880	{
10881	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
10882	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
10883	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
10884	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
10885	{
10886	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
10887	{
10888	vmxHCReadToTransient<HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE>(pVCpu, pVmxTransient);
10889	uEventArg = pVmxTransient->uExitIntErrorCode;
10890	}
10891	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
10892	switch (enmEvent1)
10893	{
10894	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
10895	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
10896	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
10897	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
10898	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
10899	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
10900	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
10901	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
10902	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
10903	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
10904	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
10905	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
10906	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
10907	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
10908	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
10909	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
10910	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
10911	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
10912	default: break;
10913	}
10914	}
10915	else
10916	AssertFailed();
10917	break;
10918
10919	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
10920	uEventArg = idxVector;
10921	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
10922	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
10923	break;
10924	}
10925	break;
10926	}
10927
10928	case VMX_EXIT_TRIPLE_FAULT:
10929	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
10930	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
10931	break;
10932	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
10933	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
10934	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
10935	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
10936	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
10937
10938	/* Instruction specific VM-exits: */
10939	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
10940	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
10941	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
10942	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
10943	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
10944	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
10945	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
10946	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
10947	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
10948	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
10949	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
10950	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
10951	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
10952	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
10953	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
10954	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
10955	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
10956	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
10957	case VMX_EXIT_MOV_CRX:
10958	vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
10959	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
10960	SET_BOTH(CRX_READ);
10961	else
10962	SET_BOTH(CRX_WRITE);
10963	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
10964	break;
10965	case VMX_EXIT_MOV_DRX:
10966	vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
10967	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
10968	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
10969	SET_BOTH(DRX_READ);
10970	else
10971	SET_BOTH(DRX_WRITE);
10972	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
10973	break;
10974	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
10975	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
10976	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
10977	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
10978	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
10979	case VMX_EXIT_GDTR_IDTR_ACCESS:
10980	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_INFO>(pVCpu, pVmxTransient);
10981	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
10982	{
10983	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
10984	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
10985	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
10986	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
10987	}
10988	break;
10989
10990	case VMX_EXIT_LDTR_TR_ACCESS:
10991	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_INFO>(pVCpu, pVmxTransient);
10992	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
10993	{
10994	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
10995	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
10996	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
10997	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
10998	}
10999	break;
11000
11001	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11002	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11003	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11004	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11005	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11006	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11007	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11008	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11009	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11010	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11011	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11012
11013	/* Events that aren't relevant at this point. */
11014	case VMX_EXIT_EXT_INT:
11015	case VMX_EXIT_INT_WINDOW:
11016	case VMX_EXIT_NMI_WINDOW:
11017	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11018	case VMX_EXIT_PREEMPT_TIMER:
11019	case VMX_EXIT_IO_INSTR:
11020	break;
11021
11022	/* Errors and unexpected events. */
11023	case VMX_EXIT_INIT_SIGNAL:
11024	case VMX_EXIT_SIPI:
11025	case VMX_EXIT_IO_SMI:
11026	case VMX_EXIT_SMI:
11027	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11028	case VMX_EXIT_ERR_MSR_LOAD:
11029	case VMX_EXIT_ERR_MACHINE_CHECK:
11030	case VMX_EXIT_PML_FULL:
11031	case VMX_EXIT_VIRTUALIZED_EOI:
11032	break;
11033
11034	default:
11035	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11036	break;
11037	}
11038	#undef SET_BOTH
11039	#undef SET_EXIT
11040
11041	/*
11042	* Dtrace tracepoints go first. We do them here at once so we don't
11043	* have to copy the guest state saving and stuff a few dozen times.
11044	* Down side is that we've got to repeat the switch, though this time
11045	* we use enmEvent since the probes are a subset of what DBGF does.
11046	*/
11047	if (fDtrace1 \|\| fDtrace2)
11048	{
11049	vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
11050	vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11051	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11052	switch (enmEvent1)
11053	{
11054	/** @todo consider which extra parameters would be helpful for each probe. */
11055	case DBGFEVENT_END: break;
11056	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11057	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11058	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11059	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11060	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11061	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11062	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11063	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11064	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
11065	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
11066	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
11067	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
11068	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
11069	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
11070	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
11071	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
11072	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
11073	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
11074	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11075	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11076	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
11077	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
11078	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
11079	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
11080	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
11081	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
11082	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
11083	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11084	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11085	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11086	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11087	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11088	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
11089	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11090	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
11091	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
11092	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
11093	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
11094	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
11095	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
11096	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
11097	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
11098	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
11099	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
11100	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
11101	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
11102	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
11103	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
11104	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
11105	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
11106	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
11107	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
11108	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
11109	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
11110	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
11111	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
11112	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
11113	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
11114	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
11115	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
11116	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
11117	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
11118	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
11119	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
11120	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
11121	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
11122	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
11123	}
11124	switch (enmEvent2)
11125	{
11126	/** @todo consider which extra parameters would be helpful for each probe. */
11127	case DBGFEVENT_END: break;
11128	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
11129	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11130	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
11131	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
11132	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
11133	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
11134	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
11135	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
11136	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
11137	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11138	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11139	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11140	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11141	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11142	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
11143	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11144	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
11145	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
11146	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
11147	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
11148	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
11149	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
11150	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
11151	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
11152	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
11153	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
11154	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
11155	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
11156	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
11157	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
11158	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
11159	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
11160	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
11161	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
11162	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
11163	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
11164	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
11165	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
11166	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
11167	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
11168	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
11169	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
11170	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
11171	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
11172	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
11173	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
11174	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
11175	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
11176	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
11177	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
11178	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
11179	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
11180	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
11181	}
11182	}
11183
11184	/*
11185	* Fire of the DBGF event, if enabled (our check here is just a quick one,
11186	* the DBGF call will do a full check).
11187	*
11188	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
11189	* Note! If we have to events, we prioritize the first, i.e. the instruction
11190	* one, in order to avoid event nesting.
11191	*/
11192	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
11193	if ( enmEvent1 != DBGFEVENT_END
11194	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
11195	{
11196	vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11197	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
11198	if (rcStrict != VINF_SUCCESS)
11199	return rcStrict;
11200	}
11201	else if ( enmEvent2 != DBGFEVENT_END
11202	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
11203	{
11204	vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11205	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
11206	if (rcStrict != VINF_SUCCESS)
11207	return rcStrict;
11208	}
11209
11210	return VINF_SUCCESS;
11211	}
11212
11213
11214	/**
11215	* Single-stepping VM-exit filtering.
11216	*
11217	* This is preprocessing the VM-exits and deciding whether we've gotten far
11218	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
11219	* handling is performed.
11220	*
11221	* @returns Strict VBox status code (i.e. informational status codes too).
11222	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
11223	* @param pVmxTransient The VMX-transient structure.
11224	* @param pDbgState The debug state.
11225	*/
11226	DECLINLINE(VBOXSTRICTRC) vmxHCRunDebugHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11227	{
11228	/*
11229	* Expensive (saves context) generic dtrace VM-exit probe.
11230	*/
11231	uint32_t const uExitReason = pVmxTransient->uExitReason;
11232	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
11233	{ /* more likely */ }
11234	else
11235	{
11236	vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
11237	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11238	AssertRC(rc);
11239	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
11240	}
11241
11242	#ifndef IN_NEM_DARWIN
11243	/*
11244	* Check for host NMI, just to get that out of the way.
11245	*/
11246	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
11247	{ /* normally likely */ }
11248	else
11249	{
11250	vmxHCReadToTransient<HMVMX_READ_EXIT_INTERRUPTION_INFO>(pVCpu, pVmxTransient);
11251	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11252	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11253	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
11254	}
11255	#endif
11256
11257	/*
11258	* Check for single stepping event if we're stepping.
11259	*/
11260	if (VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
11261	{
11262	switch (uExitReason)
11263	{
11264	case VMX_EXIT_MTF:
11265	return vmxHCExitMtf(pVCpu, pVmxTransient);
11266
11267	/* Various events: */
11268	case VMX_EXIT_XCPT_OR_NMI:
11269	case VMX_EXIT_EXT_INT:
11270	case VMX_EXIT_TRIPLE_FAULT:
11271	case VMX_EXIT_INT_WINDOW:
11272	case VMX_EXIT_NMI_WINDOW:
11273	case VMX_EXIT_TASK_SWITCH:
11274	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11275	case VMX_EXIT_APIC_ACCESS:
11276	case VMX_EXIT_EPT_VIOLATION:
11277	case VMX_EXIT_EPT_MISCONFIG:
11278	case VMX_EXIT_PREEMPT_TIMER:
11279
11280	/* Instruction specific VM-exits: */
11281	case VMX_EXIT_CPUID:
11282	case VMX_EXIT_GETSEC:
11283	case VMX_EXIT_HLT:
11284	case VMX_EXIT_INVD:
11285	case VMX_EXIT_INVLPG:
11286	case VMX_EXIT_RDPMC:
11287	case VMX_EXIT_RDTSC:
11288	case VMX_EXIT_RSM:
11289	case VMX_EXIT_VMCALL:
11290	case VMX_EXIT_VMCLEAR:
11291	case VMX_EXIT_VMLAUNCH:
11292	case VMX_EXIT_VMPTRLD:
11293	case VMX_EXIT_VMPTRST:
11294	case VMX_EXIT_VMREAD:
11295	case VMX_EXIT_VMRESUME:
11296	case VMX_EXIT_VMWRITE:
11297	case VMX_EXIT_VMXOFF:
11298	case VMX_EXIT_VMXON:
11299	case VMX_EXIT_MOV_CRX:
11300	case VMX_EXIT_MOV_DRX:
11301	case VMX_EXIT_IO_INSTR:
11302	case VMX_EXIT_RDMSR:
11303	case VMX_EXIT_WRMSR:
11304	case VMX_EXIT_MWAIT:
11305	case VMX_EXIT_MONITOR:
11306	case VMX_EXIT_PAUSE:
11307	case VMX_EXIT_GDTR_IDTR_ACCESS:
11308	case VMX_EXIT_LDTR_TR_ACCESS:
11309	case VMX_EXIT_INVEPT:
11310	case VMX_EXIT_RDTSCP:
11311	case VMX_EXIT_INVVPID:
11312	case VMX_EXIT_WBINVD:
11313	case VMX_EXIT_XSETBV:
11314	case VMX_EXIT_RDRAND:
11315	case VMX_EXIT_INVPCID:
11316	case VMX_EXIT_VMFUNC:
11317	case VMX_EXIT_RDSEED:
11318	case VMX_EXIT_XSAVES:
11319	case VMX_EXIT_XRSTORS:
11320	{
11321	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11322	AssertRCReturn(rc, rc);
11323	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
11324	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
11325	return VINF_EM_DBG_STEPPED;
11326	break;
11327	}
11328
11329	/* Errors and unexpected events: */
11330	case VMX_EXIT_INIT_SIGNAL:
11331	case VMX_EXIT_SIPI:
11332	case VMX_EXIT_IO_SMI:
11333	case VMX_EXIT_SMI:
11334	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11335	case VMX_EXIT_ERR_MSR_LOAD:
11336	case VMX_EXIT_ERR_MACHINE_CHECK:
11337	case VMX_EXIT_PML_FULL:
11338	case VMX_EXIT_VIRTUALIZED_EOI:
11339	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
11340	break;
11341
11342	default:
11343	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11344	break;
11345	}
11346	}
11347
11348	/*
11349	* Check for debugger event breakpoints and dtrace probes.
11350	*/
11351	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
11352	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
11353	{
11354	VBOXSTRICTRC rcStrict = vmxHCHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
11355	if (rcStrict != VINF_SUCCESS)
11356	return rcStrict;
11357	}
11358
11359	/*
11360	* Normal processing.
11361	*/
11362	#ifdef HMVMX_USE_FUNCTION_TABLE
11363	return g_aVMExitHandlers[uExitReason].pfn(pVCpu, pVmxTransient);
11364	#else
11365	return vmxHCHandleExit(pVCpu, pVmxTransient, uExitReason);
11366	#endif
11367	}
11368
11369	/** @} */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/VMXAllTemplate.cpp.h@ 97060

Download in other formats: