VMXAllTemplate.cpp.h@ 93115

Last change on this file since 93115 was 93115, checked in by vboxsync, 3 years ago
scm --update-copyright-year
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 606.0 KB

Line
1	/* $Id: VMXAllTemplate.cpp.h 93115 2022-01-01 11:31:46Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Code template for our own hypervisor and the NEM darwin backend using Apple's Hypervisor.framework.
4	*/
5
6	/*
7	* Copyright (C) 2012-2022 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Defined Constants And Macros *
21	*********************************************************************************************************************************/
22	#if !defined(VMX_VMCS_WRITE_16) \|\| !defined(VMX_VMCS_WRITE_32) \|\| !defined(VMX_VMCS_WRITE_64) \|\| !defined(VMX_VMCS_WRITE_64)
23	# error "At least one of the VMX_VMCS_WRITE_16, VMX_VMCS_WRITE_32, VMX_VMCS_WRITE_64 or VMX_VMCS_WRITE_64 is missing"
24	#endif
25
26
27	#if !defined(VMX_VMCS_READ_16) \|\| !defined(VMX_VMCS_READ_32) \|\| !defined(VMX_VMCS_READ_64) \|\| !defined(VMX_VMCS_READ_64)
28	# error "At least one of the VMX_VMCS_READ_16, VMX_VMCS_READ_32, VMX_VMCS_READ_64 or VMX_VMCS_READ_64 is missing"
29	#endif
30
31
32	/** Use the function table. */
33	#define HMVMX_USE_FUNCTION_TABLE
34
35	/** Determine which tagged-TLB flush handler to use. */
36	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
37	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
38	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
39	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
40
41	/**
42	* Flags to skip redundant reads of some common VMCS fields that are not part of
43	* the guest-CPU or VCPU state but are needed while handling VM-exits.
44	*/
45	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
46	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
47	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
48	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
49	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
50	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
51	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
52	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
53	#define HMVMX_READ_GUEST_PHYSICAL_ADDR RT_BIT_32(8)
54	#define HMVMX_READ_GUEST_PENDING_DBG_XCPTS RT_BIT_32(9)
55
56	/** All the VMCS fields required for processing of exception/NMI VM-exits. */
57	#define HMVMX_READ_XCPT_INFO ( HMVMX_READ_EXIT_INTERRUPTION_INFO \
58	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE \
59	\| HMVMX_READ_EXIT_INSTR_LEN \
60	\| HMVMX_READ_IDT_VECTORING_INFO \
61	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE)
62
63	/** Assert that all the given fields have been read from the VMCS. */
64	#ifdef VBOX_STRICT
65	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) \
66	do { \
67	uint32_t const fVmcsFieldRead = ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead); \
68	Assert((fVmcsFieldRead & (a_fReadFields)) == (a_fReadFields)); \
69	} while (0)
70	#else
71	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) do { } while (0)
72	#endif
73
74	/**
75	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
76	* guest using hardware-assisted VMX.
77	*
78	* This excludes state like GPRs (other than RSP) which are always are
79	* swapped and restored across the world-switch and also registers like EFER,
80	* MSR which cannot be modified by the guest without causing a VM-exit.
81	*/
82	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
83	\| CPUMCTX_EXTRN_RFLAGS \
84	\| CPUMCTX_EXTRN_RSP \
85	\| CPUMCTX_EXTRN_SREG_MASK \
86	\| CPUMCTX_EXTRN_TABLE_MASK \
87	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
88	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
89	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
90	\| CPUMCTX_EXTRN_TSC_AUX \
91	\| CPUMCTX_EXTRN_OTHER_MSRS \
92	\| CPUMCTX_EXTRN_CR0 \
93	\| CPUMCTX_EXTRN_CR3 \
94	\| CPUMCTX_EXTRN_CR4 \
95	\| CPUMCTX_EXTRN_DR7 \
96	\| CPUMCTX_EXTRN_HWVIRT \
97	\| CPUMCTX_EXTRN_INHIBIT_INT \
98	\| CPUMCTX_EXTRN_INHIBIT_NMI)
99
100	/**
101	* Exception bitmap mask for real-mode guests (real-on-v86).
102	*
103	* We need to intercept all exceptions manually except:
104	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
105	* due to bugs in Intel CPUs.
106	* - \#PF need not be intercepted even in real-mode if we have nested paging
107	* support.
108	*/
109	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
110	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
111	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
112	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
113	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
114	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
115	\| RT_BIT(X86_XCPT_XF))
116
117	/** Maximum VM-instruction error number. */
118	#define HMVMX_INSTR_ERROR_MAX 28
119
120	/** Profiling macro. */
121	#ifdef HM_PROFILE_EXIT_DISPATCH
122	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitDispatch, ed)
123	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitDispatch, ed)
124	#else
125	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
126	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
127	#endif
128
129	#ifdef IN_RING0
130	/** Assert that preemption is disabled or covered by thread-context hooks. */
131	# define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
132	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
133
134	/** Assert that we haven't migrated CPUs when thread-context hooks are not
135	* used. */
136	# define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
137	\|\| (a_pVCpu)->hmr0.s.idEnteredCpu == RTMpCpuId(), \
138	("Illegal migration! Entered on CPU %u Current %u\n", \
139	(a_pVCpu)->hmr0.s.idEnteredCpu, RTMpCpuId()))
140	#else
141	# define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) do { } while (0)
142	# define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) do { } while (0)
143	#endif
144
145	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
146	* context. */
147	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
148	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
149	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
150
151	/** Log the VM-exit reason with an easily visible marker to identify it in a
152	* potential sea of logging data. */
153	#define HMVMX_LOG_EXIT(a_pVCpu, a_uExitReason) \
154	do { \
155	Log4(("VM-exit: vcpu[%RU32] %85s -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu, \
156	HMGetVmxExitName(a_uExitReason))); \
157	} while (0) \
158
159
160	/*********************************************************************************************************************************
161	* Structures and Typedefs *
162	*********************************************************************************************************************************/
163	/**
164	* Memory operand read or write access.
165	*/
166	typedef enum VMXMEMACCESS
167	{
168	VMXMEMACCESS_READ = 0,
169	VMXMEMACCESS_WRITE = 1
170	} VMXMEMACCESS;
171
172
173	/**
174	* VMX VM-exit handler.
175	*
176	* @returns Strict VBox status code (i.e. informational status codes too).
177	* @param pVCpu The cross context virtual CPU structure.
178	* @param pVmxTransient The VMX-transient structure.
179	*/
180	#ifndef HMVMX_USE_FUNCTION_TABLE
181	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
182	#else
183	typedef DECLCALLBACKTYPE(VBOXSTRICTRC, FNVMXEXITHANDLER,(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient));
184	/** Pointer to VM-exit handler. */
185	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
186	#endif
187
188	/**
189	* VMX VM-exit handler, non-strict status code.
190	*
191	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
192	*
193	* @returns VBox status code, no informational status code returned.
194	* @param pVCpu The cross context virtual CPU structure.
195	* @param pVmxTransient The VMX-transient structure.
196	*
197	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
198	* use of that status code will be replaced with VINF_EM_SOMETHING
199	* later when switching over to IEM.
200	*/
201	#ifndef HMVMX_USE_FUNCTION_TABLE
202	typedef int FNVMXEXITHANDLERNSRC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
203	#else
204	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
205	#endif
206
207
208	/*********************************************************************************************************************************
209	* Internal Functions *
210	*********************************************************************************************************************************/
211	#ifndef HMVMX_USE_FUNCTION_TABLE
212	DECLINLINE(VBOXSTRICTRC) vmxHCHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
213	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
214	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
215	#else
216	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
217	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
218	#endif
219	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
220	DECLINLINE(VBOXSTRICTRC) vmxHCHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
221	#endif
222
223	static int vmxHCImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
224
225	/** @name VM-exit handler prototypes.
226	* @{
227	*/
228	static FNVMXEXITHANDLER vmxHCExitXcptOrNmi;
229	static FNVMXEXITHANDLER vmxHCExitExtInt;
230	static FNVMXEXITHANDLER vmxHCExitTripleFault;
231	static FNVMXEXITHANDLERNSRC vmxHCExitIntWindow;
232	static FNVMXEXITHANDLERNSRC vmxHCExitNmiWindow;
233	static FNVMXEXITHANDLER vmxHCExitTaskSwitch;
234	static FNVMXEXITHANDLER vmxHCExitCpuid;
235	static FNVMXEXITHANDLER vmxHCExitGetsec;
236	static FNVMXEXITHANDLER vmxHCExitHlt;
237	static FNVMXEXITHANDLERNSRC vmxHCExitInvd;
238	static FNVMXEXITHANDLER vmxHCExitInvlpg;
239	static FNVMXEXITHANDLER vmxHCExitRdpmc;
240	static FNVMXEXITHANDLER vmxHCExitVmcall;
241	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
242	static FNVMXEXITHANDLER vmxHCExitVmclear;
243	static FNVMXEXITHANDLER vmxHCExitVmlaunch;
244	static FNVMXEXITHANDLER vmxHCExitVmptrld;
245	static FNVMXEXITHANDLER vmxHCExitVmptrst;
246	static FNVMXEXITHANDLER vmxHCExitVmread;
247	static FNVMXEXITHANDLER vmxHCExitVmresume;
248	static FNVMXEXITHANDLER vmxHCExitVmwrite;
249	static FNVMXEXITHANDLER vmxHCExitVmxoff;
250	static FNVMXEXITHANDLER vmxHCExitVmxon;
251	static FNVMXEXITHANDLER vmxHCExitInvvpid;
252	#endif
253	static FNVMXEXITHANDLER vmxHCExitRdtsc;
254	static FNVMXEXITHANDLER vmxHCExitMovCRx;
255	static FNVMXEXITHANDLER vmxHCExitMovDRx;
256	static FNVMXEXITHANDLER vmxHCExitIoInstr;
257	static FNVMXEXITHANDLER vmxHCExitRdmsr;
258	static FNVMXEXITHANDLER vmxHCExitWrmsr;
259	static FNVMXEXITHANDLER vmxHCExitMwait;
260	static FNVMXEXITHANDLER vmxHCExitMtf;
261	static FNVMXEXITHANDLER vmxHCExitMonitor;
262	static FNVMXEXITHANDLER vmxHCExitPause;
263	static FNVMXEXITHANDLERNSRC vmxHCExitTprBelowThreshold;
264	static FNVMXEXITHANDLER vmxHCExitApicAccess;
265	static FNVMXEXITHANDLER vmxHCExitEptViolation;
266	static FNVMXEXITHANDLER vmxHCExitEptMisconfig;
267	static FNVMXEXITHANDLER vmxHCExitRdtscp;
268	static FNVMXEXITHANDLER vmxHCExitPreemptTimer;
269	static FNVMXEXITHANDLERNSRC vmxHCExitWbinvd;
270	static FNVMXEXITHANDLER vmxHCExitXsetbv;
271	static FNVMXEXITHANDLER vmxHCExitInvpcid;
272	static FNVMXEXITHANDLERNSRC vmxHCExitSetPendingXcptUD;
273	static FNVMXEXITHANDLERNSRC vmxHCExitErrInvalidGuestState;
274	static FNVMXEXITHANDLERNSRC vmxHCExitErrUnexpected;
275	/** @} */
276
277	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
278	/** @name Nested-guest VM-exit handler prototypes.
279	* @{
280	*/
281	static FNVMXEXITHANDLER vmxHCExitXcptOrNmiNested;
282	static FNVMXEXITHANDLER vmxHCExitTripleFaultNested;
283	static FNVMXEXITHANDLERNSRC vmxHCExitIntWindowNested;
284	static FNVMXEXITHANDLERNSRC vmxHCExitNmiWindowNested;
285	static FNVMXEXITHANDLER vmxHCExitTaskSwitchNested;
286	static FNVMXEXITHANDLER vmxHCExitHltNested;
287	static FNVMXEXITHANDLER vmxHCExitInvlpgNested;
288	static FNVMXEXITHANDLER vmxHCExitRdpmcNested;
289	static FNVMXEXITHANDLER vmxHCExitVmreadVmwriteNested;
290	static FNVMXEXITHANDLER vmxHCExitRdtscNested;
291	static FNVMXEXITHANDLER vmxHCExitMovCRxNested;
292	static FNVMXEXITHANDLER vmxHCExitMovDRxNested;
293	static FNVMXEXITHANDLER vmxHCExitIoInstrNested;
294	static FNVMXEXITHANDLER vmxHCExitRdmsrNested;
295	static FNVMXEXITHANDLER vmxHCExitWrmsrNested;
296	static FNVMXEXITHANDLER vmxHCExitMwaitNested;
297	static FNVMXEXITHANDLER vmxHCExitMtfNested;
298	static FNVMXEXITHANDLER vmxHCExitMonitorNested;
299	static FNVMXEXITHANDLER vmxHCExitPauseNested;
300	static FNVMXEXITHANDLERNSRC vmxHCExitTprBelowThresholdNested;
301	static FNVMXEXITHANDLER vmxHCExitApicAccessNested;
302	static FNVMXEXITHANDLER vmxHCExitApicWriteNested;
303	static FNVMXEXITHANDLER vmxHCExitVirtEoiNested;
304	static FNVMXEXITHANDLER vmxHCExitRdtscpNested;
305	static FNVMXEXITHANDLERNSRC vmxHCExitWbinvdNested;
306	static FNVMXEXITHANDLER vmxHCExitInvpcidNested;
307	static FNVMXEXITHANDLERNSRC vmxHCExitErrInvalidGuestStateNested;
308	static FNVMXEXITHANDLER vmxHCExitInstrNested;
309	static FNVMXEXITHANDLER vmxHCExitInstrWithInfoNested;
310	/** @} */
311	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
312
313
314	/*********************************************************************************************************************************
315	* Global Variables *
316	*********************************************************************************************************************************/
317	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
318	/**
319	* Array of all VMCS fields.
320	* Any fields added to the VT-x spec. should be added here.
321	*
322	* Currently only used to derive shadow VMCS fields for hardware-assisted execution
323	* of nested-guests.
324	*/
325	static const uint32_t g_aVmcsFields[] =
326	{
327	/* 16-bit control fields. */
328	VMX_VMCS16_VPID,
329	VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR,
330	VMX_VMCS16_EPTP_INDEX,
331
332	/* 16-bit guest-state fields. */
333	VMX_VMCS16_GUEST_ES_SEL,
334	VMX_VMCS16_GUEST_CS_SEL,
335	VMX_VMCS16_GUEST_SS_SEL,
336	VMX_VMCS16_GUEST_DS_SEL,
337	VMX_VMCS16_GUEST_FS_SEL,
338	VMX_VMCS16_GUEST_GS_SEL,
339	VMX_VMCS16_GUEST_LDTR_SEL,
340	VMX_VMCS16_GUEST_TR_SEL,
341	VMX_VMCS16_GUEST_INTR_STATUS,
342	VMX_VMCS16_GUEST_PML_INDEX,
343
344	/* 16-bits host-state fields. */
345	VMX_VMCS16_HOST_ES_SEL,
346	VMX_VMCS16_HOST_CS_SEL,
347	VMX_VMCS16_HOST_SS_SEL,
348	VMX_VMCS16_HOST_DS_SEL,
349	VMX_VMCS16_HOST_FS_SEL,
350	VMX_VMCS16_HOST_GS_SEL,
351	VMX_VMCS16_HOST_TR_SEL,
352
353	/* 64-bit control fields. */
354	VMX_VMCS64_CTRL_IO_BITMAP_A_FULL,
355	VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH,
356	VMX_VMCS64_CTRL_IO_BITMAP_B_FULL,
357	VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH,
358	VMX_VMCS64_CTRL_MSR_BITMAP_FULL,
359	VMX_VMCS64_CTRL_MSR_BITMAP_HIGH,
360	VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL,
361	VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH,
362	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,
363	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH,
364	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL,
365	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH,
366	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL,
367	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH,
368	VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL,
369	VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH,
370	VMX_VMCS64_CTRL_TSC_OFFSET_FULL,
371	VMX_VMCS64_CTRL_TSC_OFFSET_HIGH,
372	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL,
373	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH,
374	VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL,
375	VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH,
376	VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL,
377	VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH,
378	VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL,
379	VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH,
380	VMX_VMCS64_CTRL_EPTP_FULL,
381	VMX_VMCS64_CTRL_EPTP_HIGH,
382	VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL,
383	VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH,
384	VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL,
385	VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH,
386	VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL,
387	VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH,
388	VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL,
389	VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH,
390	VMX_VMCS64_CTRL_EPTP_LIST_FULL,
391	VMX_VMCS64_CTRL_EPTP_LIST_HIGH,
392	VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL,
393	VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH,
394	VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL,
395	VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH,
396	VMX_VMCS64_CTRL_VE_XCPT_INFO_ADDR_FULL,
397	VMX_VMCS64_CTRL_VE_XCPT_INFO_ADDR_HIGH,
398	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL,
399	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH,
400	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL,
401	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH,
402	VMX_VMCS64_CTRL_SPPTP_FULL,
403	VMX_VMCS64_CTRL_SPPTP_HIGH,
404	VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL,
405	VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH,
406	VMX_VMCS64_CTRL_PROC_EXEC3_FULL,
407	VMX_VMCS64_CTRL_PROC_EXEC3_HIGH,
408	VMX_VMCS64_CTRL_ENCLV_EXITING_BITMAP_FULL,
409	VMX_VMCS64_CTRL_ENCLV_EXITING_BITMAP_HIGH,
410
411	/* 64-bit read-only data fields. */
412	VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,
413	VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH,
414
415	/* 64-bit guest-state fields. */
416	VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL,
417	VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH,
418	VMX_VMCS64_GUEST_DEBUGCTL_FULL,
419	VMX_VMCS64_GUEST_DEBUGCTL_HIGH,
420	VMX_VMCS64_GUEST_PAT_FULL,
421	VMX_VMCS64_GUEST_PAT_HIGH,
422	VMX_VMCS64_GUEST_EFER_FULL,
423	VMX_VMCS64_GUEST_EFER_HIGH,
424	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL,
425	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH,
426	VMX_VMCS64_GUEST_PDPTE0_FULL,
427	VMX_VMCS64_GUEST_PDPTE0_HIGH,
428	VMX_VMCS64_GUEST_PDPTE1_FULL,
429	VMX_VMCS64_GUEST_PDPTE1_HIGH,
430	VMX_VMCS64_GUEST_PDPTE2_FULL,
431	VMX_VMCS64_GUEST_PDPTE2_HIGH,
432	VMX_VMCS64_GUEST_PDPTE3_FULL,
433	VMX_VMCS64_GUEST_PDPTE3_HIGH,
434	VMX_VMCS64_GUEST_BNDCFGS_FULL,
435	VMX_VMCS64_GUEST_BNDCFGS_HIGH,
436	VMX_VMCS64_GUEST_RTIT_CTL_FULL,
437	VMX_VMCS64_GUEST_RTIT_CTL_HIGH,
438	VMX_VMCS64_GUEST_PKRS_FULL,
439	VMX_VMCS64_GUEST_PKRS_HIGH,
440
441	/* 64-bit host-state fields. */
442	VMX_VMCS64_HOST_PAT_FULL,
443	VMX_VMCS64_HOST_PAT_HIGH,
444	VMX_VMCS64_HOST_EFER_FULL,
445	VMX_VMCS64_HOST_EFER_HIGH,
446	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL,
447	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH,
448	VMX_VMCS64_HOST_PKRS_FULL,
449	VMX_VMCS64_HOST_PKRS_HIGH,
450
451	/* 32-bit control fields. */
452	VMX_VMCS32_CTRL_PIN_EXEC,
453	VMX_VMCS32_CTRL_PROC_EXEC,
454	VMX_VMCS32_CTRL_EXCEPTION_BITMAP,
455	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,
456	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,
457	VMX_VMCS32_CTRL_CR3_TARGET_COUNT,
458	VMX_VMCS32_CTRL_EXIT,
459	VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,
460	VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,
461	VMX_VMCS32_CTRL_ENTRY,
462	VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,
463	VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,
464	VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,
465	VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,
466	VMX_VMCS32_CTRL_TPR_THRESHOLD,
467	VMX_VMCS32_CTRL_PROC_EXEC2,
468	VMX_VMCS32_CTRL_PLE_GAP,
469	VMX_VMCS32_CTRL_PLE_WINDOW,
470
471	/* 32-bits read-only fields. */
472	VMX_VMCS32_RO_VM_INSTR_ERROR,
473	VMX_VMCS32_RO_EXIT_REASON,
474	VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,
475	VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE,
476	VMX_VMCS32_RO_IDT_VECTORING_INFO,
477	VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,
478	VMX_VMCS32_RO_EXIT_INSTR_LENGTH,
479	VMX_VMCS32_RO_EXIT_INSTR_INFO,
480
481	/* 32-bit guest-state fields. */
482	VMX_VMCS32_GUEST_ES_LIMIT,
483	VMX_VMCS32_GUEST_CS_LIMIT,
484	VMX_VMCS32_GUEST_SS_LIMIT,
485	VMX_VMCS32_GUEST_DS_LIMIT,
486	VMX_VMCS32_GUEST_FS_LIMIT,
487	VMX_VMCS32_GUEST_GS_LIMIT,
488	VMX_VMCS32_GUEST_LDTR_LIMIT,
489	VMX_VMCS32_GUEST_TR_LIMIT,
490	VMX_VMCS32_GUEST_GDTR_LIMIT,
491	VMX_VMCS32_GUEST_IDTR_LIMIT,
492	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
493	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
494	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
495	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
496	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
497	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS,
498	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS,
499	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS,
500	VMX_VMCS32_GUEST_INT_STATE,
501	VMX_VMCS32_GUEST_ACTIVITY_STATE,
502	VMX_VMCS32_GUEST_SMBASE,
503	VMX_VMCS32_GUEST_SYSENTER_CS,
504	VMX_VMCS32_PREEMPT_TIMER_VALUE,
505
506	/* 32-bit host-state fields. */
507	VMX_VMCS32_HOST_SYSENTER_CS,
508
509	/* Natural-width control fields. */
510	VMX_VMCS_CTRL_CR0_MASK,
511	VMX_VMCS_CTRL_CR4_MASK,
512	VMX_VMCS_CTRL_CR0_READ_SHADOW,
513	VMX_VMCS_CTRL_CR4_READ_SHADOW,
514	VMX_VMCS_CTRL_CR3_TARGET_VAL0,
515	VMX_VMCS_CTRL_CR3_TARGET_VAL1,
516	VMX_VMCS_CTRL_CR3_TARGET_VAL2,
517	VMX_VMCS_CTRL_CR3_TARGET_VAL3,
518
519	/* Natural-width read-only data fields. */
520	VMX_VMCS_RO_EXIT_QUALIFICATION,
521	VMX_VMCS_RO_IO_RCX,
522	VMX_VMCS_RO_IO_RSI,
523	VMX_VMCS_RO_IO_RDI,
524	VMX_VMCS_RO_IO_RIP,
525	VMX_VMCS_RO_GUEST_LINEAR_ADDR,
526
527	/* Natural-width guest-state field */
528	VMX_VMCS_GUEST_CR0,
529	VMX_VMCS_GUEST_CR3,
530	VMX_VMCS_GUEST_CR4,
531	VMX_VMCS_GUEST_ES_BASE,
532	VMX_VMCS_GUEST_CS_BASE,
533	VMX_VMCS_GUEST_SS_BASE,
534	VMX_VMCS_GUEST_DS_BASE,
535	VMX_VMCS_GUEST_FS_BASE,
536	VMX_VMCS_GUEST_GS_BASE,
537	VMX_VMCS_GUEST_LDTR_BASE,
538	VMX_VMCS_GUEST_TR_BASE,
539	VMX_VMCS_GUEST_GDTR_BASE,
540	VMX_VMCS_GUEST_IDTR_BASE,
541	VMX_VMCS_GUEST_DR7,
542	VMX_VMCS_GUEST_RSP,
543	VMX_VMCS_GUEST_RIP,
544	VMX_VMCS_GUEST_RFLAGS,
545	VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
546	VMX_VMCS_GUEST_SYSENTER_ESP,
547	VMX_VMCS_GUEST_SYSENTER_EIP,
548	VMX_VMCS_GUEST_S_CET,
549	VMX_VMCS_GUEST_SSP,
550	VMX_VMCS_GUEST_INTR_SSP_TABLE_ADDR,
551
552	/* Natural-width host-state fields */
553	VMX_VMCS_HOST_CR0,
554	VMX_VMCS_HOST_CR3,
555	VMX_VMCS_HOST_CR4,
556	VMX_VMCS_HOST_FS_BASE,
557	VMX_VMCS_HOST_GS_BASE,
558	VMX_VMCS_HOST_TR_BASE,
559	VMX_VMCS_HOST_GDTR_BASE,
560	VMX_VMCS_HOST_IDTR_BASE,
561	VMX_VMCS_HOST_SYSENTER_ESP,
562	VMX_VMCS_HOST_SYSENTER_EIP,
563	VMX_VMCS_HOST_RSP,
564	VMX_VMCS_HOST_RIP,
565	VMX_VMCS_HOST_S_CET,
566	VMX_VMCS_HOST_SSP,
567	VMX_VMCS_HOST_INTR_SSP_TABLE_ADDR
568	};
569	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
570
571	#ifdef VBOX_STRICT
572	static const uint32_t g_aVmcsSegBase[] =
573	{
574	VMX_VMCS_GUEST_ES_BASE,
575	VMX_VMCS_GUEST_CS_BASE,
576	VMX_VMCS_GUEST_SS_BASE,
577	VMX_VMCS_GUEST_DS_BASE,
578	VMX_VMCS_GUEST_FS_BASE,
579	VMX_VMCS_GUEST_GS_BASE
580	};
581	static const uint32_t g_aVmcsSegSel[] =
582	{
583	VMX_VMCS16_GUEST_ES_SEL,
584	VMX_VMCS16_GUEST_CS_SEL,
585	VMX_VMCS16_GUEST_SS_SEL,
586	VMX_VMCS16_GUEST_DS_SEL,
587	VMX_VMCS16_GUEST_FS_SEL,
588	VMX_VMCS16_GUEST_GS_SEL
589	};
590	static const uint32_t g_aVmcsSegLimit[] =
591	{
592	VMX_VMCS32_GUEST_ES_LIMIT,
593	VMX_VMCS32_GUEST_CS_LIMIT,
594	VMX_VMCS32_GUEST_SS_LIMIT,
595	VMX_VMCS32_GUEST_DS_LIMIT,
596	VMX_VMCS32_GUEST_FS_LIMIT,
597	VMX_VMCS32_GUEST_GS_LIMIT
598	};
599	static const uint32_t g_aVmcsSegAttr[] =
600	{
601	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
602	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
603	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
604	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
605	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
606	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
607	};
608	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
609	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
610	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
611	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
612	#endif /* VBOX_STRICT */
613
614	#ifdef HMVMX_USE_FUNCTION_TABLE
615	/**
616	* VMX_EXIT dispatch table.
617	*/
618	static const struct CLANG11NOTHROWWEIRDNESS { PFNVMXEXITHANDLER pfn; } g_aVMExitHandlers[VMX_EXIT_MAX + 1] =
619	{
620	/* 0 VMX_EXIT_XCPT_OR_NMI */ { vmxHCExitXcptOrNmi },
621	/* 1 VMX_EXIT_EXT_INT */ { vmxHCExitExtInt },
622	/* 2 VMX_EXIT_TRIPLE_FAULT */ { vmxHCExitTripleFault },
623	/* 3 VMX_EXIT_INIT_SIGNAL */ { vmxHCExitErrUnexpected },
624	/* 4 VMX_EXIT_SIPI */ { vmxHCExitErrUnexpected },
625	/* 5 VMX_EXIT_IO_SMI */ { vmxHCExitErrUnexpected },
626	/* 6 VMX_EXIT_SMI */ { vmxHCExitErrUnexpected },
627	/* 7 VMX_EXIT_INT_WINDOW */ { vmxHCExitIntWindow },
628	/* 8 VMX_EXIT_NMI_WINDOW */ { vmxHCExitNmiWindow },
629	/* 9 VMX_EXIT_TASK_SWITCH */ { vmxHCExitTaskSwitch },
630	/* 10 VMX_EXIT_CPUID */ { vmxHCExitCpuid },
631	/* 11 VMX_EXIT_GETSEC */ { vmxHCExitGetsec },
632	/* 12 VMX_EXIT_HLT */ { vmxHCExitHlt },
633	/* 13 VMX_EXIT_INVD */ { vmxHCExitInvd },
634	/* 14 VMX_EXIT_INVLPG */ { vmxHCExitInvlpg },
635	/* 15 VMX_EXIT_RDPMC */ { vmxHCExitRdpmc },
636	/* 16 VMX_EXIT_RDTSC */ { vmxHCExitRdtsc },
637	/* 17 VMX_EXIT_RSM */ { vmxHCExitErrUnexpected },
638	/* 18 VMX_EXIT_VMCALL */ { vmxHCExitVmcall },
639	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
640	/* 19 VMX_EXIT_VMCLEAR */ { vmxHCExitVmclear },
641	/* 20 VMX_EXIT_VMLAUNCH */ { vmxHCExitVmlaunch },
642	/* 21 VMX_EXIT_VMPTRLD */ { vmxHCExitVmptrld },
643	/* 22 VMX_EXIT_VMPTRST */ { vmxHCExitVmptrst },
644	/* 23 VMX_EXIT_VMREAD */ { vmxHCExitVmread },
645	/* 24 VMX_EXIT_VMRESUME */ { vmxHCExitVmresume },
646	/* 25 VMX_EXIT_VMWRITE */ { vmxHCExitVmwrite },
647	/* 26 VMX_EXIT_VMXOFF */ { vmxHCExitVmxoff },
648	/* 27 VMX_EXIT_VMXON */ { vmxHCExitVmxon },
649	#else
650	/* 19 VMX_EXIT_VMCLEAR */ { vmxHCExitSetPendingXcptUD },
651	/* 20 VMX_EXIT_VMLAUNCH */ { vmxHCExitSetPendingXcptUD },
652	/* 21 VMX_EXIT_VMPTRLD */ { vmxHCExitSetPendingXcptUD },
653	/* 22 VMX_EXIT_VMPTRST */ { vmxHCExitSetPendingXcptUD },
654	/* 23 VMX_EXIT_VMREAD */ { vmxHCExitSetPendingXcptUD },
655	/* 24 VMX_EXIT_VMRESUME */ { vmxHCExitSetPendingXcptUD },
656	/* 25 VMX_EXIT_VMWRITE */ { vmxHCExitSetPendingXcptUD },
657	/* 26 VMX_EXIT_VMXOFF */ { vmxHCExitSetPendingXcptUD },
658	/* 27 VMX_EXIT_VMXON */ { vmxHCExitSetPendingXcptUD },
659	#endif
660	/* 28 VMX_EXIT_MOV_CRX */ { vmxHCExitMovCRx },
661	/* 29 VMX_EXIT_MOV_DRX */ { vmxHCExitMovDRx },
662	/* 30 VMX_EXIT_IO_INSTR */ { vmxHCExitIoInstr },
663	/* 31 VMX_EXIT_RDMSR */ { vmxHCExitRdmsr },
664	/* 32 VMX_EXIT_WRMSR */ { vmxHCExitWrmsr },
665	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ { vmxHCExitErrInvalidGuestState },
666	/* 34 VMX_EXIT_ERR_MSR_LOAD */ { vmxHCExitErrUnexpected },
667	/* 35 UNDEFINED */ { vmxHCExitErrUnexpected },
668	/* 36 VMX_EXIT_MWAIT */ { vmxHCExitMwait },
669	/* 37 VMX_EXIT_MTF */ { vmxHCExitMtf },
670	/* 38 UNDEFINED */ { vmxHCExitErrUnexpected },
671	/* 39 VMX_EXIT_MONITOR */ { vmxHCExitMonitor },
672	/* 40 VMX_EXIT_PAUSE */ { vmxHCExitPause },
673	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ { vmxHCExitErrUnexpected },
674	/* 42 UNDEFINED */ { vmxHCExitErrUnexpected },
675	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ { vmxHCExitTprBelowThreshold },
676	/* 44 VMX_EXIT_APIC_ACCESS */ { vmxHCExitApicAccess },
677	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ { vmxHCExitErrUnexpected },
678	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ { vmxHCExitErrUnexpected },
679	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ { vmxHCExitErrUnexpected },
680	/* 48 VMX_EXIT_EPT_VIOLATION */ { vmxHCExitEptViolation },
681	/* 49 VMX_EXIT_EPT_MISCONFIG */ { vmxHCExitEptMisconfig },
682	/* 50 VMX_EXIT_INVEPT */ { vmxHCExitSetPendingXcptUD },
683	/* 51 VMX_EXIT_RDTSCP */ { vmxHCExitRdtscp },
684	/* 52 VMX_EXIT_PREEMPT_TIMER */ { vmxHCExitPreemptTimer },
685	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
686	/* 53 VMX_EXIT_INVVPID */ { vmxHCExitInvvpid },
687	#else
688	/* 53 VMX_EXIT_INVVPID */ { vmxHCExitSetPendingXcptUD },
689	#endif
690	/* 54 VMX_EXIT_WBINVD */ { vmxHCExitWbinvd },
691	/* 55 VMX_EXIT_XSETBV */ { vmxHCExitXsetbv },
692	/* 56 VMX_EXIT_APIC_WRITE */ { vmxHCExitErrUnexpected },
693	/* 57 VMX_EXIT_RDRAND */ { vmxHCExitErrUnexpected },
694	/* 58 VMX_EXIT_INVPCID */ { vmxHCExitInvpcid },
695	/* 59 VMX_EXIT_VMFUNC */ { vmxHCExitErrUnexpected },
696	/* 60 VMX_EXIT_ENCLS */ { vmxHCExitErrUnexpected },
697	/* 61 VMX_EXIT_RDSEED */ { vmxHCExitErrUnexpected },
698	/* 62 VMX_EXIT_PML_FULL */ { vmxHCExitErrUnexpected },
699	/* 63 VMX_EXIT_XSAVES */ { vmxHCExitErrUnexpected },
700	/* 64 VMX_EXIT_XRSTORS */ { vmxHCExitErrUnexpected },
701	/* 65 UNDEFINED */ { vmxHCExitErrUnexpected },
702	/* 66 VMX_EXIT_SPP_EVENT */ { vmxHCExitErrUnexpected },
703	/* 67 VMX_EXIT_UMWAIT */ { vmxHCExitErrUnexpected },
704	/* 68 VMX_EXIT_TPAUSE */ { vmxHCExitErrUnexpected },
705	/* 69 VMX_EXIT_LOADIWKEY */ { vmxHCExitErrUnexpected },
706	};
707	#endif /* HMVMX_USE_FUNCTION_TABLE */
708
709	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
710	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
711	{
712	/* 0 */ "(Not Used)",
713	/* 1 */ "VMCALL executed in VMX root operation.",
714	/* 2 */ "VMCLEAR with invalid physical address.",
715	/* 3 */ "VMCLEAR with VMXON pointer.",
716	/* 4 */ "VMLAUNCH with non-clear VMCS.",
717	/* 5 */ "VMRESUME with non-launched VMCS.",
718	/* 6 */ "VMRESUME after VMXOFF",
719	/* 7 */ "VM-entry with invalid control fields.",
720	/* 8 */ "VM-entry with invalid host state fields.",
721	/* 9 */ "VMPTRLD with invalid physical address.",
722	/* 10 */ "VMPTRLD with VMXON pointer.",
723	/* 11 */ "VMPTRLD with incorrect revision identifier.",
724	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
725	/* 13 */ "VMWRITE to read-only VMCS component.",
726	/* 14 */ "(Not Used)",
727	/* 15 */ "VMXON executed in VMX root operation.",
728	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
729	/* 17 */ "VM-entry with non-launched executing VMCS.",
730	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
731	/* 19 */ "VMCALL with non-clear VMCS.",
732	/* 20 */ "VMCALL with invalid VM-exit control fields.",
733	/* 21 */ "(Not Used)",
734	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
735	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
736	/* 24 */ "VMCALL with invalid SMM-monitor features.",
737	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
738	/* 26 */ "VM-entry with events blocked by MOV SS.",
739	/* 27 */ "(Not Used)",
740	/* 28 */ "Invalid operand to INVEPT/INVVPID."
741	};
742	#endif /* VBOX_STRICT && LOG_ENABLED */
743
744
745	#ifdef IN_RING0
746	/**
747	* Checks if the given MSR is part of the lastbranch-from-IP MSR stack.
748	* @returns @c true if it's part of LBR stack, @c false otherwise.
749	*
750	* @param pVM The cross context VM structure.
751	* @param idMsr The MSR.
752	* @param pidxMsr Where to store the index of the MSR in the LBR MSR array.
753	* Optional, can be NULL.
754	*
755	* @remarks Must only be called when LBR is enabled.
756	*/
757	DECL_FORCE_INLINE(bool) vmxHCIsLbrBranchFromMsr(PCVMCC pVM, uint32_t idMsr, uint32_t *pidxMsr)
758	{
759	Assert(VM_IS_VMX_LBR(pVM));
760	Assert(pVM->hmr0.s.vmx.idLbrFromIpMsrFirst);
761	uint32_t const cLbrStack = pVM->hmr0.s.vmx.idLbrFromIpMsrLast - pVM->hmr0.s.vmx.idLbrFromIpMsrFirst + 1;
762	uint32_t const idxMsr = idMsr - pVM->hmr0.s.vmx.idLbrFromIpMsrFirst;
763	if (idxMsr < cLbrStack)
764	{
765	if (pidxMsr)
766	*pidxMsr = idxMsr;
767	return true;
768	}
769	return false;
770	}
771
772
773	/**
774	* Checks if the given MSR is part of the lastbranch-to-IP MSR stack.
775	* @returns @c true if it's part of LBR stack, @c false otherwise.
776	*
777	* @param pVM The cross context VM structure.
778	* @param idMsr The MSR.
779	* @param pidxMsr Where to store the index of the MSR in the LBR MSR array.
780	* Optional, can be NULL.
781	*
782	* @remarks Must only be called when LBR is enabled and when lastbranch-to-IP MSRs
783	* are supported by the CPU (see vmxHCSetupLbrMsrRange).
784	*/
785	DECL_FORCE_INLINE(bool) vmxHCIsLbrBranchToMsr(PCVMCC pVM, uint32_t idMsr, uint32_t *pidxMsr)
786	{
787	Assert(VM_IS_VMX_LBR(pVM));
788	if (pVM->hmr0.s.vmx.idLbrToIpMsrFirst)
789	{
790	uint32_t const cLbrStack = pVM->hmr0.s.vmx.idLbrToIpMsrLast - pVM->hmr0.s.vmx.idLbrToIpMsrFirst + 1;
791	uint32_t const idxMsr = idMsr - pVM->hmr0.s.vmx.idLbrToIpMsrFirst;
792	if (idxMsr < cLbrStack)
793	{
794	if (pidxMsr)
795	*pidxMsr = idxMsr;
796	return true;
797	}
798	}
799	return false;
800	}
801	#endif
802
803
804	/**
805	* Gets the CR0 guest/host mask.
806	*
807	* These bits typically does not change through the lifetime of a VM. Any bit set in
808	* this mask is owned by the host/hypervisor and would cause a VM-exit when modified
809	* by the guest.
810	*
811	* @returns The CR0 guest/host mask.
812	* @param pVCpu The cross context virtual CPU structure.
813	*/
814	static uint64_t vmxHCGetFixedCr0Mask(PCVMCPUCC pVCpu)
815	{
816	/*
817	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
818	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
819	*
820	* Furthermore, modifications to any bits that are reserved/unspecified currently
821	* by the Intel spec. must also cause a VM-exit. This prevents unpredictable behavior
822	* when future CPUs specify and use currently reserved/unspecified bits.
823	*/
824	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
825	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
826	* and @bugref{6944}. */
827	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
828	return ( X86_CR0_PE
829	\| X86_CR0_NE
830	\| (VM_IS_VMX_NESTED_PAGING(pVM) ? 0 : X86_CR0_WP)
831	\| X86_CR0_PG
832	\| VMX_EXIT_HOST_CR0_IGNORE_MASK);
833	}
834
835
836	/**
837	* Gets the CR4 guest/host mask.
838	*
839	* These bits typically does not change through the lifetime of a VM. Any bit set in
840	* this mask is owned by the host/hypervisor and would cause a VM-exit when modified
841	* by the guest.
842	*
843	* @returns The CR4 guest/host mask.
844	* @param pVCpu The cross context virtual CPU structure.
845	*/
846	static uint64_t vmxHCGetFixedCr4Mask(PCVMCPUCC pVCpu)
847	{
848	/*
849	* We construct a mask of all CR4 bits that the guest can modify without causing
850	* a VM-exit. Then invert this mask to obtain all CR4 bits that should cause
851	* a VM-exit when the guest attempts to modify them when executing using
852	* hardware-assisted VMX.
853	*
854	* When a feature is not exposed to the guest (and may be present on the host),
855	* we want to intercept guest modifications to the bit so we can emulate proper
856	* behavior (e.g., #GP).
857	*
858	* Furthermore, only modifications to those bits that don't require immediate
859	* emulation is allowed. For e.g., PCIDE is excluded because the behavior
860	* depends on CR3 which might not always be the guest value while executing
861	* using hardware-assisted VMX.
862	*/
863	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
864	bool const fFsGsBase = pVM->cpum.ro.GuestFeatures.fFsGsBase;
865	bool const fXSaveRstor = pVM->cpum.ro.GuestFeatures.fXSaveRstor;
866	bool const fFxSaveRstor = pVM->cpum.ro.GuestFeatures.fFxSaveRstor;
867
868	/*
869	* Paranoia.
870	* Ensure features exposed to the guest are present on the host.
871	*/
872	Assert(!fFsGsBase \|\| pVM->cpum.ro.HostFeatures.fFsGsBase);
873	Assert(!fXSaveRstor \|\| pVM->cpum.ro.HostFeatures.fXSaveRstor);
874	Assert(!fFxSaveRstor \|\| pVM->cpum.ro.HostFeatures.fFxSaveRstor);
875
876	uint64_t const fGstMask = ( X86_CR4_PVI
877	\| X86_CR4_TSD
878	\| X86_CR4_DE
879	\| X86_CR4_MCE
880	\| X86_CR4_PCE
881	\| X86_CR4_OSXMMEEXCPT
882	\| (fFsGsBase ? X86_CR4_FSGSBASE : 0)
883	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
884	\| (fFxSaveRstor ? X86_CR4_OSFXSR : 0));
885	return ~fGstMask;
886	}
887
888
889	/**
890	* Returns whether the VM-exit MSR-store area differs from the VM-exit MSR-load
891	* area.
892	*
893	* @returns @c true if it's different, @c false otherwise.
894	* @param pVmcsInfo The VMCS info. object.
895	*/
896	DECL_FORCE_INLINE(bool) vmxHCIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
897	{
898	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
899	&& pVmcsInfo->pvGuestMsrStore);
900	}
901
902	#ifdef IN_RING0
903	/**
904	* Sets the given Processor-based VM-execution controls.
905	*
906	* @param pVCpu The cross context virtual CPU structure.
907	* @param pVmxTransient The VMX-transient structure.
908	* @param uProcCtls The Processor-based VM-execution controls to set.
909	*/
910	static void vmxHCSetProcCtlsVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
911	{
912	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
913	if ((pVmcsInfo->u32ProcCtls & uProcCtls) != uProcCtls)
914	{
915	pVmcsInfo->u32ProcCtls \|= uProcCtls;
916	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
917	AssertRC(rc);
918	}
919	}
920
921
922	/**
923	* Removes the given Processor-based VM-execution controls.
924	*
925	* @param pVCpu The cross context virtual CPU structure.
926	* @param pVmxTransient The VMX-transient structure.
927	* @param uProcCtls The Processor-based VM-execution controls to remove.
928	*
929	* @remarks When executing a nested-guest, this will not remove any of the specified
930	* controls if the nested hypervisor has set any one of them.
931	*/
932	static void vmxHCRemoveProcCtlsVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
933	{
934	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
935	if (pVmcsInfo->u32ProcCtls & uProcCtls)
936	{
937	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
938	if ( !pVmxTransient->fIsNestedGuest
939	\|\| !CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, uProcCtls))
940	#else
941	NOREF(pVCpu);
942	if (!pVmxTransient->fIsNestedGuest)
943	#endif
944	{
945	pVmcsInfo->u32ProcCtls &= ~uProcCtls;
946	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
947	AssertRC(rc);
948	}
949	}
950	}
951
952
953	/**
954	* Sets the TSC offset for the current VMCS.
955	*
956	* @param pVCpu The cross context virtual CPU structure.
957	* @param uTscOffset The TSC offset to set.
958	* @param pVmcsInfo The VMCS info. object.
959	*/
960	static void vmxHCSetTscOffsetVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t uTscOffset)
961	{
962	if (pVmcsInfo->u64TscOffset != uTscOffset)
963	{
964	int rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
965	AssertRC(rc);
966	pVmcsInfo->u64TscOffset = uTscOffset;
967	}
968	}
969	#endif
970
971	/**
972	* Adds one or more exceptions to the exception bitmap and commits it to the current
973	* VMCS.
974	*
975	* @param pVCpu The cross context virtual CPU structure.
976	* @param pVmxTransient The VMX-transient structure.
977	* @param uXcptMask The exception(s) to add.
978	*/
979	static void vmxHCAddXcptInterceptMask(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
980	{
981	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
982	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
983	if ((uXcptBitmap & uXcptMask) != uXcptMask)
984	{
985	uXcptBitmap \|= uXcptMask;
986	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
987	AssertRC(rc);
988	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
989	}
990	}
991
992
993	/**
994	* Adds an exception to the exception bitmap and commits it to the current VMCS.
995	*
996	* @param pVCpu The cross context virtual CPU structure.
997	* @param pVmxTransient The VMX-transient structure.
998	* @param uXcpt The exception to add.
999	*/
1000	static void vmxHCAddXcptIntercept(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1001	{
1002	Assert(uXcpt <= X86_XCPT_LAST);
1003	vmxHCAddXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT_32(uXcpt));
1004	}
1005
1006
1007	/**
1008	* Remove one or more exceptions from the exception bitmap and commits it to the
1009	* current VMCS.
1010	*
1011	* This takes care of not removing the exception intercept if a nested-guest
1012	* requires the exception to be intercepted.
1013	*
1014	* @returns VBox status code.
1015	* @param pVCpu The cross context virtual CPU structure.
1016	* @param pVmxTransient The VMX-transient structure.
1017	* @param uXcptMask The exception(s) to remove.
1018	*/
1019	static int vmxHCRemoveXcptInterceptMask(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1020	{
1021	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1022	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
1023	if (u32XcptBitmap & uXcptMask)
1024	{
1025	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1026	if (!pVmxTransient->fIsNestedGuest)
1027	{ /* likely */ }
1028	else
1029	uXcptMask &= ~pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32XcptBitmap;
1030	#endif
1031	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
1032	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
1033	\| RT_BIT(X86_XCPT_DE)
1034	\| RT_BIT(X86_XCPT_NM)
1035	\| RT_BIT(X86_XCPT_TS)
1036	\| RT_BIT(X86_XCPT_UD)
1037	\| RT_BIT(X86_XCPT_NP)
1038	\| RT_BIT(X86_XCPT_SS)
1039	\| RT_BIT(X86_XCPT_GP)
1040	\| RT_BIT(X86_XCPT_PF)
1041	\| RT_BIT(X86_XCPT_MF));
1042	#elif defined(HMVMX_ALWAYS_TRAP_PF)
1043	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
1044	#endif
1045	if (uXcptMask)
1046	{
1047	/* Validate we are not removing any essential exception intercepts. */
1048	#ifdef IN_RING0
1049	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF)));
1050	#else
1051	Assert(!(uXcptMask & RT_BIT(X86_XCPT_PF)));
1052	#endif
1053	NOREF(pVCpu);
1054	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
1055	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
1056
1057	/* Remove it from the exception bitmap. */
1058	u32XcptBitmap &= ~uXcptMask;
1059
1060	/* Commit and update the cache if necessary. */
1061	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
1062	{
1063	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1064	AssertRC(rc);
1065	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
1066	}
1067	}
1068	}
1069	return VINF_SUCCESS;
1070	}
1071
1072
1073	/**
1074	* Remove an exceptions from the exception bitmap and commits it to the current
1075	* VMCS.
1076	*
1077	* @returns VBox status code.
1078	* @param pVCpu The cross context virtual CPU structure.
1079	* @param pVmxTransient The VMX-transient structure.
1080	* @param uXcpt The exception to remove.
1081	*/
1082	static int vmxHCRemoveXcptIntercept(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1083	{
1084	return vmxHCRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
1085	}
1086
1087
1088	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1089	/**
1090	* Loads the shadow VMCS specified by the VMCS info. object.
1091	*
1092	* @returns VBox status code.
1093	* @param pVmcsInfo The VMCS info. object.
1094	*
1095	* @remarks Can be called with interrupts disabled.
1096	*/
1097	static int vmxHCLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1098	{
1099	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1100	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1101
1102	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs);
1103	if (RT_SUCCESS(rc))
1104	pVmcsInfo->fShadowVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1105	return rc;
1106	}
1107
1108
1109	/**
1110	* Clears the shadow VMCS specified by the VMCS info. object.
1111	*
1112	* @returns VBox status code.
1113	* @param pVmcsInfo The VMCS info. object.
1114	*
1115	* @remarks Can be called with interrupts disabled.
1116	*/
1117	static int vmxHCClearShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1118	{
1119	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1120	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1121
1122	int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs);
1123	if (RT_SUCCESS(rc))
1124	pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1125	return rc;
1126	}
1127
1128
1129	/**
1130	* Switches from and to the specified VMCSes.
1131	*
1132	* @returns VBox status code.
1133	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
1134	* @param pVmcsInfoTo The VMCS info. object we are switching to.
1135	*
1136	* @remarks Called with interrupts disabled.
1137	*/
1138	static int vmxHCSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
1139	{
1140	/*
1141	* Clear the VMCS we are switching out if it has not already been cleared.
1142	* This will sync any CPU internal data back to the VMCS.
1143	*/
1144	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1145	{
1146	int rc = vmxHCClearVmcs(pVmcsInfoFrom);
1147	if (RT_SUCCESS(rc))
1148	{
1149	/*
1150	* The shadow VMCS, if any, would not be active at this point since we
1151	* would have cleared it while importing the virtual hardware-virtualization
1152	* state as part the VMLAUNCH/VMRESUME VM-exit. Hence, there's no need to
1153	* clear the shadow VMCS here, just assert for safety.
1154	*/
1155	Assert(!pVmcsInfoFrom->pvShadowVmcs \|\| pVmcsInfoFrom->fShadowVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR);
1156	}
1157	else
1158	return rc;
1159	}
1160
1161	/*
1162	* Clear the VMCS we are switching to if it has not already been cleared.
1163	* This will initialize the VMCS launch state to "clear" required for loading it.
1164	*
1165	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
1166	*/
1167	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1168	{
1169	int rc = vmxHCClearVmcs(pVmcsInfoTo);
1170	if (RT_SUCCESS(rc))
1171	{ /* likely */ }
1172	else
1173	return rc;
1174	}
1175
1176	/*
1177	* Finally, load the VMCS we are switching to.
1178	*/
1179	return vmxHCLoadVmcs(pVmcsInfoTo);
1180	}
1181
1182
1183	/**
1184	* Switches between the guest VMCS and the nested-guest VMCS as specified by the
1185	* caller.
1186	*
1187	* @returns VBox status code.
1188	* @param pVCpu The cross context virtual CPU structure.
1189	* @param fSwitchToNstGstVmcs Whether to switch to the nested-guest VMCS (pass
1190	* true) or guest VMCS (pass false).
1191	*/
1192	static int vmxHCSwitchToGstOrNstGstVmcs(PVMCPUCC pVCpu, bool fSwitchToNstGstVmcs)
1193	{
1194	/* Ensure we have synced everything from the guest-CPU context to the VMCS before switching. */
1195	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
1196
1197	PVMXVMCSINFO pVmcsInfoFrom;
1198	PVMXVMCSINFO pVmcsInfoTo;
1199	if (fSwitchToNstGstVmcs)
1200	{
1201	pVmcsInfoFrom = &pVCpu->hmr0.s.vmx.VmcsInfo;
1202	pVmcsInfoTo = &pVCpu->hmr0.s.vmx.VmcsInfoNstGst;
1203	}
1204	else
1205	{
1206	pVmcsInfoFrom = &pVCpu->hmr0.s.vmx.VmcsInfoNstGst;
1207	pVmcsInfoTo = &pVCpu->hmr0.s.vmx.VmcsInfo;
1208	}
1209
1210	/*
1211	* Disable interrupts to prevent being preempted while we switch the current VMCS as the
1212	* preemption hook code path acquires the current VMCS.
1213	*/
1214	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1215
1216	int rc = vmxHCSwitchVmcs(pVmcsInfoFrom, pVmcsInfoTo);
1217	if (RT_SUCCESS(rc))
1218	{
1219	pVCpu->hmr0.s.vmx.fSwitchedToNstGstVmcs = fSwitchToNstGstVmcs;
1220	VCPU_2_VMXSTATE(pVCpu).vmx.fSwitchedToNstGstVmcsCopyForRing3 = fSwitchToNstGstVmcs;
1221
1222	/*
1223	* If we are switching to a VMCS that was executed on a different host CPU or was
1224	* never executed before, flag that we need to export the host state before executing
1225	* guest/nested-guest code using hardware-assisted VMX.
1226	*
1227	* This could probably be done in a preemptible context since the preemption hook
1228	* will flag the necessary change in host context. However, since preemption is
1229	* already disabled and to avoid making assumptions about host specific code in
1230	* RTMpCpuId when called with preemption enabled, we'll do this while preemption is
1231	* disabled.
1232	*/
1233	if (pVmcsInfoTo->idHostCpuState == RTMpCpuId())
1234	{ /* likely */ }
1235	else
1236	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE);
1237
1238	ASMSetFlags(fEFlags);
1239
1240	/*
1241	* We use a different VM-exit MSR-store areas for the guest and nested-guest. Hence,
1242	* flag that we need to update the host MSR values there. Even if we decide in the
1243	* future to share the VM-exit MSR-store area page between the guest and nested-guest,
1244	* if its content differs, we would have to update the host MSRs anyway.
1245	*/
1246	pVCpu->hmr0.s.vmx.fUpdatedHostAutoMsrs = false;
1247	}
1248	else
1249	ASMSetFlags(fEFlags);
1250	return rc;
1251	}
1252	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
1253
1254
1255	#ifdef IN_RING0
1256	/**
1257	* Updates the VM's last error record.
1258	*
1259	* If there was a VMX instruction error, reads the error data from the VMCS and
1260	* updates VCPU's last error record as well.
1261	*
1262	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1263	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
1264	* VERR_VMX_INVALID_VMCS_FIELD.
1265	* @param rc The error code.
1266	*/
1267	static void vmxHCUpdateErrorRecord(PVMCPUCC pVCpu, int rc)
1268	{
1269	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
1270	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
1271	{
1272	AssertPtrReturnVoid(pVCpu);
1273	VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_VM_INSTR_ERROR, &VCPU_2_VMXSTATE(pVCpu).vmx.LastError.u32InstrError);
1274	}
1275	pVCpu->CTX_SUFF(pVM)->hm.s.ForR3.rcInit = rc;
1276	}
1277	#endif
1278
1279
1280	#ifdef VBOX_STRICT
1281	/**
1282	* Reads the VM-entry interruption-information field from the VMCS into the VMX
1283	* transient structure.
1284	*
1285	* @param pVCpu The cross context virtual CPU structure.
1286	* @param pVmxTransient The VMX-transient structure.
1287	*/
1288	DECLINLINE(void) vmxHCReadEntryIntInfoVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1289	{
1290	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
1291	AssertRC(rc);
1292	}
1293
1294
1295	/**
1296	* Reads the VM-entry exception error code field from the VMCS into
1297	* the VMX transient structure.
1298	*
1299	* @param pVCpu The cross context virtual CPU structure.
1300	* @param pVmxTransient The VMX-transient structure.
1301	*/
1302	DECLINLINE(void) vmxHCReadEntryXcptErrorCodeVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1303	{
1304	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1305	AssertRC(rc);
1306	}
1307
1308
1309	/**
1310	* Reads the VM-entry exception error code field from the VMCS into
1311	* the VMX transient structure.
1312	*
1313	* @param pVCpu The cross context virtual CPU structure.
1314	* @param pVmxTransient The VMX-transient structure.
1315	*/
1316	DECLINLINE(void) vmxHCReadEntryInstrLenVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1317	{
1318	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1319	AssertRC(rc);
1320	}
1321	#endif /* VBOX_STRICT */
1322
1323
1324	/**
1325	* Reads the VM-exit interruption-information field from the VMCS into the VMX
1326	* transient structure.
1327	*
1328	* @param pVCpu The cross context virtual CPU structure.
1329	* @param pVmxTransient The VMX-transient structure.
1330	*/
1331	DECLINLINE(void) vmxHCReadExitIntInfoVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1332	{
1333	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1334	{
1335	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1336	AssertRC(rc);
1337	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
1338	}
1339	}
1340
1341
1342	/**
1343	* Reads the VM-exit interruption error code from the VMCS into the VMX
1344	* transient structure.
1345	*
1346	* @param pVCpu The cross context virtual CPU structure.
1347	* @param pVmxTransient The VMX-transient structure.
1348	*/
1349	DECLINLINE(void) vmxHCReadExitIntErrorCodeVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1350	{
1351	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1352	{
1353	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1354	AssertRC(rc);
1355	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1356	}
1357	}
1358
1359
1360	/**
1361	* Reads the VM-exit instruction length field from the VMCS into the VMX
1362	* transient structure.
1363	*
1364	* @param pVCpu The cross context virtual CPU structure.
1365	* @param pVmxTransient The VMX-transient structure.
1366	*/
1367	DECLINLINE(void) vmxHCReadExitInstrLenVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1368	{
1369	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1370	{
1371	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1372	AssertRC(rc);
1373	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1374	}
1375	}
1376
1377
1378	/**
1379	* Reads the VM-exit instruction-information field from the VMCS into
1380	* the VMX transient structure.
1381	*
1382	* @param pVCpu The cross context virtual CPU structure.
1383	* @param pVmxTransient The VMX-transient structure.
1384	*/
1385	DECLINLINE(void) vmxHCReadExitInstrInfoVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1386	{
1387	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1388	{
1389	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1390	AssertRC(rc);
1391	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1392	}
1393	}
1394
1395
1396	/**
1397	* Reads the Exit Qualification from the VMCS into the VMX transient structure.
1398	*
1399	* @param pVCpu The cross context virtual CPU structure.
1400	* @param pVmxTransient The VMX-transient structure.
1401	*/
1402	DECLINLINE(void) vmxHCReadExitQualVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1403	{
1404	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1405	{
1406	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1407	AssertRC(rc);
1408	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1409	}
1410	}
1411
1412
1413	/**
1414	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1415	*
1416	* @param pVCpu The cross context virtual CPU structure.
1417	* @param pVmxTransient The VMX-transient structure.
1418	*/
1419	DECLINLINE(void) vmxHCReadGuestLinearAddrVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1420	{
1421	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1422	{
1423	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1424	AssertRC(rc);
1425	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1426	}
1427	}
1428
1429
1430	/**
1431	* Reads the Guest-physical address from the VMCS into the VMX transient structure.
1432	*
1433	* @param pVCpu The cross context virtual CPU structure.
1434	* @param pVmxTransient The VMX-transient structure.
1435	*/
1436	DECLINLINE(void) vmxHCReadGuestPhysicalAddrVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1437	{
1438	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_PHYSICAL_ADDR))
1439	{
1440	int rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1441	AssertRC(rc);
1442	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_PHYSICAL_ADDR;
1443	}
1444	}
1445
1446	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1447	/**
1448	* Reads the Guest pending-debug exceptions from the VMCS into the VMX transient
1449	* structure.
1450	*
1451	* @param pVCpu The cross context virtual CPU structure.
1452	* @param pVmxTransient The VMX-transient structure.
1453	*/
1454	DECLINLINE(void) vmxHCReadGuestPendingDbgXctps(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1455	{
1456	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_PENDING_DBG_XCPTS))
1457	{
1458	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &pVmxTransient->uGuestPendingDbgXcpts);
1459	AssertRC(rc);
1460	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_PENDING_DBG_XCPTS;
1461	}
1462	}
1463	#endif
1464
1465	/**
1466	* Reads the IDT-vectoring information field from the VMCS into the VMX
1467	* transient structure.
1468	*
1469	* @param pVCpu The cross context virtual CPU structure.
1470	* @param pVmxTransient The VMX-transient structure.
1471	*
1472	* @remarks No-long-jump zone!!!
1473	*/
1474	DECLINLINE(void) vmxHCReadIdtVectoringInfoVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1475	{
1476	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1477	{
1478	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1479	AssertRC(rc);
1480	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1481	}
1482	}
1483
1484
1485	/**
1486	* Reads the IDT-vectoring error code from the VMCS into the VMX
1487	* transient structure.
1488	*
1489	* @param pVCpu The cross context virtual CPU structure.
1490	* @param pVmxTransient The VMX-transient structure.
1491	*/
1492	DECLINLINE(void) vmxHCReadIdtVectoringErrorCodeVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1493	{
1494	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1495	{
1496	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1497	AssertRC(rc);
1498	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1499	}
1500	}
1501
1502	#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
1503	/**
1504	* Reads all relevant read-only VMCS fields into the VMX transient structure.
1505	*
1506	* @param pVCpu The cross context virtual CPU structure.
1507	* @param pVmxTransient The VMX-transient structure.
1508	*/
1509	static void vmxHCReadAllRoFieldsVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1510	{
1511	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1512	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1513	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1514	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1515	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1516	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1517	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1518	rc \|= VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1519	rc \|= VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1520	AssertRC(rc);
1521	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION
1522	\| HMVMX_READ_EXIT_INSTR_LEN
1523	\| HMVMX_READ_EXIT_INSTR_INFO
1524	\| HMVMX_READ_IDT_VECTORING_INFO
1525	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
1526	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
1527	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
1528	\| HMVMX_READ_GUEST_LINEAR_ADDR
1529	\| HMVMX_READ_GUEST_PHYSICAL_ADDR;
1530	}
1531	#endif
1532
1533	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1534	/**
1535	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
1536	*
1537	* @returns @c true if the MSR is intercepted, @c false otherwise.
1538	* @param pbMsrBitmap The MSR bitmap.
1539	* @param offMsr The MSR byte offset.
1540	* @param iBit The bit offset from the byte offset.
1541	*/
1542	DECLINLINE(bool) vmxHCIsMsrBitSet(uint8_t const *pbMsrBitmap, uint16_t offMsr, int32_t iBit)
1543	{
1544	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
1545	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
1546	}
1547	#endif
1548
1549	#ifdef IN_RING0
1550	/**
1551	* Sets the permission bits for the specified MSR in the given MSR bitmap.
1552	*
1553	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
1554	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
1555	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
1556	* the read/write access of this MSR.
1557	*
1558	* @param pVCpu The cross context virtual CPU structure.
1559	* @param pVmcsInfo The VMCS info. object.
1560	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1561	* @param idMsr The MSR value.
1562	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
1563	* include both a read -and- a write permission!
1564	*
1565	* @sa CPUMGetVmxMsrPermission.
1566	* @remarks Can be called with interrupts disabled.
1567	*/
1568	static void vmxHCSetMsrPermission(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
1569	{
1570	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
1571	Assert(pbMsrBitmap);
1572	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
1573
1574	/*
1575	* MSR-bitmap Layout:
1576	* Byte index MSR range Interpreted as
1577	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1578	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1579	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1580	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1581	*
1582	* A bit corresponding to an MSR within the above range causes a VM-exit
1583	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
1584	* the MSR range, it always cause a VM-exit.
1585	*
1586	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
1587	*/
1588	uint16_t const offBitmapRead = 0;
1589	uint16_t const offBitmapWrite = 0x800;
1590	uint16_t offMsr;
1591	int32_t iBit;
1592	if (idMsr <= UINT32_C(0x00001fff))
1593	{
1594	offMsr = 0;
1595	iBit = idMsr;
1596	}
1597	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1598	{
1599	offMsr = 0x400;
1600	iBit = idMsr - UINT32_C(0xc0000000);
1601	}
1602	else
1603	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
1604
1605	/*
1606	* Set the MSR read permission.
1607	*/
1608	uint16_t const offMsrRead = offBitmapRead + offMsr;
1609	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
1610	if (fMsrpm & VMXMSRPM_ALLOW_RD)
1611	{
1612	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1613	bool const fClear = !fIsNstGstVmcs ? true
1614	: !vmxHCIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap, offMsrRead, iBit);
1615	#else
1616	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1617	bool const fClear = true;
1618	#endif
1619	if (fClear)
1620	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
1621	}
1622	else
1623	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
1624
1625	/*
1626	* Set the MSR write permission.
1627	*/
1628	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
1629	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
1630	if (fMsrpm & VMXMSRPM_ALLOW_WR)
1631	{
1632	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1633	bool const fClear = !fIsNstGstVmcs ? true
1634	: !vmxHCIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap, offMsrWrite, iBit);
1635	#else
1636	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1637	bool const fClear = true;
1638	#endif
1639	if (fClear)
1640	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
1641	}
1642	else
1643	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
1644	}
1645
1646
1647	/**
1648	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1649	* area.
1650	*
1651	* @returns VBox status code.
1652	* @param pVCpu The cross context virtual CPU structure.
1653	* @param pVmcsInfo The VMCS info. object.
1654	* @param cMsrs The number of MSRs.
1655	*/
1656	static int vmxHCSetAutoLoadStoreMsrCount(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
1657	{
1658	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1659	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(g_HmMsrs.u.vmx.u64Misc);
1660	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
1661	{
1662	/* Commit the MSR counts to the VMCS and update the cache. */
1663	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
1664	{
1665	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRC(rc);
1666	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRC(rc);
1667	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRC(rc);
1668	pVmcsInfo->cEntryMsrLoad = cMsrs;
1669	pVmcsInfo->cExitMsrStore = cMsrs;
1670	pVmcsInfo->cExitMsrLoad = cMsrs;
1671	}
1672	return VINF_SUCCESS;
1673	}
1674
1675	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
1676	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1677	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1678	}
1679
1680
1681	/**
1682	* Adds a new (or updates the value of an existing) guest/host MSR
1683	* pair to be swapped during the world-switch as part of the
1684	* auto-load/store MSR area in the VMCS.
1685	*
1686	* @returns VBox status code.
1687	* @param pVCpu The cross context virtual CPU structure.
1688	* @param pVmxTransient The VMX-transient structure.
1689	* @param idMsr The MSR.
1690	* @param uGuestMsrValue Value of the guest MSR.
1691	* @param fSetReadWrite Whether to set the guest read/write access of this
1692	* MSR (thus not causing a VM-exit).
1693	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1694	* necessary.
1695	*/
1696	static int vmxHCAddAutoLoadStoreMsr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
1697	bool fSetReadWrite, bool fUpdateHostMsr)
1698	{
1699	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1700	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1701	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1702	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1703	uint32_t i;
1704
1705	/* Paranoia. */
1706	Assert(pGuestMsrLoad);
1707
1708	#ifndef DEBUG_bird
1709	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGuestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
1710	#endif
1711
1712	/* Check if the MSR already exists in the VM-entry MSR-load area. */
1713	for (i = 0; i < cMsrs; i++)
1714	{
1715	if (pGuestMsrLoad[i].u32Msr == idMsr)
1716	break;
1717	}
1718
1719	bool fAdded = false;
1720	if (i == cMsrs)
1721	{
1722	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
1723	++cMsrs;
1724	int rc = vmxHCSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1725	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
1726
1727	/* Set the guest to read/write this MSR without causing VM-exits. */
1728	if ( fSetReadWrite
1729	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
1730	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
1731
1732	Log4Func(("Added MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
1733	fAdded = true;
1734	}
1735
1736	/* Update the MSR value for the newly added or already existing MSR. */
1737	pGuestMsrLoad[i].u32Msr = idMsr;
1738	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
1739
1740	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
1741	if (vmxHCIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1742	{
1743	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1744	pGuestMsrStore[i].u32Msr = idMsr;
1745	pGuestMsrStore[i].u64Value = uGuestMsrValue;
1746	}
1747
1748	/* Update the corresponding slot in the host MSR area. */
1749	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1750	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
1751	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
1752	pHostMsr[i].u32Msr = idMsr;
1753
1754	/*
1755	* Only if the caller requests to update the host MSR value AND we've newly added the
1756	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
1757	* updated by vmxHCUpdateAutoLoadHostMsrs().
1758	*
1759	* We do this for performance reasons since reading MSRs may be quite expensive.
1760	*/
1761	if (fAdded)
1762	{
1763	if (fUpdateHostMsr)
1764	{
1765	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1766	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1767	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
1768	}
1769	else
1770	{
1771	/* Someone else can do the work. */
1772	pVCpu->hmr0.s.vmx.fUpdatedHostAutoMsrs = false;
1773	}
1774	}
1775
1776	return VINF_SUCCESS;
1777	}
1778
1779
1780	/**
1781	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1782	* auto-load/store MSR area in the VMCS.
1783	*
1784	* @returns VBox status code.
1785	* @param pVCpu The cross context virtual CPU structure.
1786	* @param pVmxTransient The VMX-transient structure.
1787	* @param idMsr The MSR.
1788	*/
1789	static int vmxHCRemoveAutoLoadStoreMsr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t idMsr)
1790	{
1791	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1792	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1793	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1794	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1795
1796	#ifndef DEBUG_bird
1797	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
1798	#endif
1799
1800	for (uint32_t i = 0; i < cMsrs; i++)
1801	{
1802	/* Find the MSR. */
1803	if (pGuestMsrLoad[i].u32Msr == idMsr)
1804	{
1805	/*
1806	* If it's the last MSR, we only need to reduce the MSR count.
1807	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
1808	*/
1809	if (i < cMsrs - 1)
1810	{
1811	/* Remove it from the VM-entry MSR-load area. */
1812	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
1813	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
1814
1815	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
1816	if (vmxHCIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1817	{
1818	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1819	Assert(pGuestMsrStore[i].u32Msr == idMsr);
1820	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
1821	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
1822	}
1823
1824	/* Remove it from the VM-exit MSR-load area. */
1825	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1826	Assert(pHostMsr[i].u32Msr == idMsr);
1827	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
1828	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
1829	}
1830
1831	/* Reduce the count to reflect the removed MSR and bail. */
1832	--cMsrs;
1833	break;
1834	}
1835	}
1836
1837	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
1838	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
1839	{
1840	int rc = vmxHCSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1841	AssertRCReturn(rc, rc);
1842
1843	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1844	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1845	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
1846
1847	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
1848	return VINF_SUCCESS;
1849	}
1850
1851	return VERR_NOT_FOUND;
1852	}
1853
1854
1855	/**
1856	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
1857	*
1858	* @returns @c true if found, @c false otherwise.
1859	* @param pVmcsInfo The VMCS info. object.
1860	* @param idMsr The MSR to find.
1861	*/
1862	static bool vmxHCIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
1863	{
1864	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1865	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
1866	Assert(pMsrs);
1867	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
1868	for (uint32_t i = 0; i < cMsrs; i++)
1869	{
1870	if (pMsrs[i].u32Msr == idMsr)
1871	return true;
1872	}
1873	return false;
1874	}
1875	#endif
1876
1877
1878	/**
1879	* Verifies that our cached values of the VMCS fields are all consistent with
1880	* what's actually present in the VMCS.
1881	*
1882	* @returns VBox status code.
1883	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
1884	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
1885	* VMCS content. HMCPU error-field is
1886	* updated, see VMX_VCI_XXX.
1887	* @param pVCpu The cross context virtual CPU structure.
1888	* @param pVmcsInfo The VMCS info. object.
1889	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1890	*/
1891	static int vmxHCCheckCachedVmcsCtls(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1892	{
1893	const char * const pcszVmcs = fIsNstGstVmcs ? "Nested-guest VMCS" : "VMCS";
1894
1895	uint32_t u32Val;
1896	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY, &u32Val);
1897	AssertRC(rc);
1898	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
1899	("%s entry controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32EntryCtls, u32Val),
1900	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_ENTRY,
1901	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1902
1903	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_EXIT, &u32Val);
1904	AssertRC(rc);
1905	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
1906	("%s exit controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ExitCtls, u32Val),
1907	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_EXIT,
1908	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1909
1910	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1911	AssertRC(rc);
1912	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
1913	("%s pin controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32PinCtls, u32Val),
1914	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PIN_EXEC,
1915	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1916
1917	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1918	AssertRC(rc);
1919	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
1920	("%s proc controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls, u32Val),
1921	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PROC_EXEC,
1922	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1923
1924	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
1925	{
1926	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1927	AssertRC(rc);
1928	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
1929	("%s proc2 controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls2, u32Val),
1930	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
1931	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1932	}
1933
1934	uint64_t u64Val;
1935	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TERTIARY_CTLS)
1936	{
1937	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_CTRL_PROC_EXEC3_FULL, &u64Val);
1938	AssertRC(rc);
1939	AssertMsgReturnStmt(pVmcsInfo->u64ProcCtls3 == u64Val,
1940	("%s proc3 controls mismatch: Cache=%#RX32 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64ProcCtls3, u64Val),
1941	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PROC_EXEC3,
1942	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1943	}
1944
1945	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
1946	AssertRC(rc);
1947	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
1948	("%s exception bitmap mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32XcptBitmap, u32Val),
1949	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
1950	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1951
1952	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
1953	AssertRC(rc);
1954	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
1955	("%s TSC offset mismatch: Cache=%#RX64 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64TscOffset, u64Val),
1956	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
1957	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1958
1959	NOREF(pcszVmcs);
1960	return VINF_SUCCESS;
1961	}
1962
1963
1964	#ifdef IN_RING0
1965	/**
1966	* Sets up the LBR MSR ranges based on the host CPU.
1967	*
1968	* @returns VBox status code.
1969	* @param pVM The cross context VM structure.
1970	*/
1971	static int vmxHCSetupLbrMsrRange(PVMCC pVM)
1972	{
1973	Assert(VM_IS_VMX_LBR(pVM));
1974	uint32_t idLbrFromIpMsrFirst;
1975	uint32_t idLbrFromIpMsrLast;
1976	uint32_t idLbrToIpMsrFirst;
1977	uint32_t idLbrToIpMsrLast;
1978	uint32_t idLbrTosMsr;
1979
1980	/*
1981	* Determine the LBR MSRs supported for this host CPU family and model.
1982	*
1983	* See Intel spec. 17.4.8 "LBR Stack".
1984	* See Intel "Model-Specific Registers" spec.
1985	*/
1986	uint32_t const uFamilyModel = (pVM->cpum.ro.HostFeatures.uFamily << 8)
1987	\| pVM->cpum.ro.HostFeatures.uModel;
1988	switch (uFamilyModel)
1989	{
1990	case 0x0f01: case 0x0f02:
1991	idLbrFromIpMsrFirst = MSR_P4_LASTBRANCH_0;
1992	idLbrFromIpMsrLast = MSR_P4_LASTBRANCH_3;
1993	idLbrToIpMsrFirst = 0x0;
1994	idLbrToIpMsrLast = 0x0;
1995	idLbrTosMsr = MSR_P4_LASTBRANCH_TOS;
1996	break;
1997
1998	case 0x065c: case 0x065f: case 0x064e: case 0x065e: case 0x068e:
1999	case 0x069e: case 0x0655: case 0x0666: case 0x067a: case 0x0667:
2000	case 0x066a: case 0x066c: case 0x067d: case 0x067e:
2001	idLbrFromIpMsrFirst = MSR_LASTBRANCH_0_FROM_IP;
2002	idLbrFromIpMsrLast = MSR_LASTBRANCH_31_FROM_IP;
2003	idLbrToIpMsrFirst = MSR_LASTBRANCH_0_TO_IP;
2004	idLbrToIpMsrLast = MSR_LASTBRANCH_31_TO_IP;
2005	idLbrTosMsr = MSR_LASTBRANCH_TOS;
2006	break;
2007
2008	case 0x063d: case 0x0647: case 0x064f: case 0x0656: case 0x063c:
2009	case 0x0645: case 0x0646: case 0x063f: case 0x062a: case 0x062d:
2010	case 0x063a: case 0x063e: case 0x061a: case 0x061e: case 0x061f:
2011	case 0x062e: case 0x0625: case 0x062c: case 0x062f:
2012	idLbrFromIpMsrFirst = MSR_LASTBRANCH_0_FROM_IP;
2013	idLbrFromIpMsrLast = MSR_LASTBRANCH_15_FROM_IP;
2014	idLbrToIpMsrFirst = MSR_LASTBRANCH_0_TO_IP;
2015	idLbrToIpMsrLast = MSR_LASTBRANCH_15_TO_IP;
2016	idLbrTosMsr = MSR_LASTBRANCH_TOS;
2017	break;
2018
2019	case 0x0617: case 0x061d: case 0x060f:
2020	idLbrFromIpMsrFirst = MSR_CORE2_LASTBRANCH_0_FROM_IP;
2021	idLbrFromIpMsrLast = MSR_CORE2_LASTBRANCH_3_FROM_IP;
2022	idLbrToIpMsrFirst = MSR_CORE2_LASTBRANCH_0_TO_IP;
2023	idLbrToIpMsrLast = MSR_CORE2_LASTBRANCH_3_TO_IP;
2024	idLbrTosMsr = MSR_CORE2_LASTBRANCH_TOS;
2025	break;
2026
2027	/* Atom and related microarchitectures we don't care about:
2028	case 0x0637: case 0x064a: case 0x064c: case 0x064d: case 0x065a:
2029	case 0x065d: case 0x061c: case 0x0626: case 0x0627: case 0x0635:
2030	case 0x0636: */
2031	/* All other CPUs: */
2032	default:
2033	{
2034	LogRelFunc(("Could not determine LBR stack size for the CPU model %#x\n", uFamilyModel));
2035	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_LBR_STACK_SIZE_UNKNOWN;
2036	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2037	}
2038	}
2039
2040	/*
2041	* Validate.
2042	*/
2043	uint32_t const cLbrStack = idLbrFromIpMsrLast - idLbrFromIpMsrFirst + 1;
2044	PCVMCPU pVCpu0 = VMCC_GET_CPU_0(pVM);
2045	AssertCompile( RT_ELEMENTS(pVCpu0->hm.s.vmx.VmcsInfo.au64LbrFromIpMsr)
2046	== RT_ELEMENTS(pVCpu0->hm.s.vmx.VmcsInfo.au64LbrToIpMsr));
2047	if (cLbrStack > RT_ELEMENTS(pVCpu0->hm.s.vmx.VmcsInfo.au64LbrFromIpMsr))
2048	{
2049	LogRelFunc(("LBR stack size of the CPU (%u) exceeds our buffer size\n", cLbrStack));
2050	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_LBR_STACK_SIZE_OVERFLOW;
2051	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2052	}
2053	NOREF(pVCpu0);
2054
2055	/*
2056	* Update the LBR info. to the VM struct. for use later.
2057	*/
2058	pVM->hmr0.s.vmx.idLbrTosMsr = idLbrTosMsr;
2059
2060	pVM->hm.s.ForR3.vmx.idLbrFromIpMsrFirst = pVM->hmr0.s.vmx.idLbrFromIpMsrFirst = idLbrFromIpMsrFirst;
2061	pVM->hm.s.ForR3.vmx.idLbrFromIpMsrLast = pVM->hmr0.s.vmx.idLbrFromIpMsrLast = idLbrFromIpMsrLast;
2062
2063	pVM->hm.s.ForR3.vmx.idLbrToIpMsrFirst = pVM->hmr0.s.vmx.idLbrToIpMsrFirst = idLbrToIpMsrFirst;
2064	pVM->hm.s.ForR3.vmx.idLbrToIpMsrLast = pVM->hmr0.s.vmx.idLbrToIpMsrLast = idLbrToIpMsrLast;
2065	return VINF_SUCCESS;
2066	}
2067
2068
2069	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2070	/**
2071	* Sets up the shadow VMCS fields arrays.
2072	*
2073	* This function builds arrays of VMCS fields to sync the shadow VMCS later while
2074	* executing the guest.
2075	*
2076	* @returns VBox status code.
2077	* @param pVM The cross context VM structure.
2078	*/
2079	static int vmxHCSetupShadowVmcsFieldsArrays(PVMCC pVM)
2080	{
2081	/*
2082	* Paranoia. Ensure we haven't exposed the VMWRITE-All VMX feature to the guest
2083	* when the host does not support it.
2084	*/
2085	bool const fGstVmwriteAll = pVM->cpum.ro.GuestFeatures.fVmxVmwriteAll;
2086	if ( !fGstVmwriteAll
2087	\|\| (g_HmMsrs.u.vmx.u64Misc & VMX_MISC_VMWRITE_ALL))
2088	{ /* likely. */ }
2089	else
2090	{
2091	LogRelFunc(("VMX VMWRITE-All feature exposed to the guest but host CPU does not support it!\n"));
2092	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_GST_HOST_VMWRITE_ALL;
2093	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2094	}
2095
2096	uint32_t const cVmcsFields = RT_ELEMENTS(g_aVmcsFields);
2097	uint32_t cRwFields = 0;
2098	uint32_t cRoFields = 0;
2099	for (uint32_t i = 0; i < cVmcsFields; i++)
2100	{
2101	VMXVMCSFIELD VmcsField;
2102	VmcsField.u = g_aVmcsFields[i];
2103
2104	/*
2105	* We will be writing "FULL" (64-bit) fields while syncing the shadow VMCS.
2106	* Therefore, "HIGH" (32-bit portion of 64-bit) fields must not be included
2107	* in the shadow VMCS fields array as they would be redundant.
2108	*
2109	* If the VMCS field depends on a CPU feature that is not exposed to the guest,
2110	* we must not include it in the shadow VMCS fields array. Guests attempting to
2111	* VMREAD/VMWRITE such VMCS fields would cause a VM-exit and we shall emulate
2112	* the required behavior.
2113	*/
2114	if ( VmcsField.n.fAccessType == VMX_VMCSFIELD_ACCESS_FULL
2115	&& CPUMIsGuestVmxVmcsFieldValid(pVM, VmcsField.u))
2116	{
2117	/*
2118	* Read-only fields are placed in a separate array so that while syncing shadow
2119	* VMCS fields later (which is more performance critical) we can avoid branches.
2120	*
2121	* However, if the guest can write to all fields (including read-only fields),
2122	* we treat it a as read/write field. Otherwise, writing to these fields would
2123	* cause a VMWRITE instruction error while syncing the shadow VMCS.
2124	*/
2125	if ( fGstVmwriteAll
2126	\|\| !VMXIsVmcsFieldReadOnly(VmcsField.u))
2127	pVM->hmr0.s.vmx.paShadowVmcsFields[cRwFields++] = VmcsField.u;
2128	else
2129	pVM->hmr0.s.vmx.paShadowVmcsRoFields[cRoFields++] = VmcsField.u;
2130	}
2131	}
2132
2133	/* Update the counts. */
2134	pVM->hmr0.s.vmx.cShadowVmcsFields = cRwFields;
2135	pVM->hmr0.s.vmx.cShadowVmcsRoFields = cRoFields;
2136	return VINF_SUCCESS;
2137	}
2138
2139
2140	/**
2141	* Sets up the VMREAD and VMWRITE bitmaps.
2142	*
2143	* @param pVM The cross context VM structure.
2144	*/
2145	static void vmxHCSetupVmreadVmwriteBitmaps(PVMCC pVM)
2146	{
2147	/*
2148	* By default, ensure guest attempts to access any VMCS fields cause VM-exits.
2149	*/
2150	uint32_t const cbBitmap = X86_PAGE_4K_SIZE;
2151	uint8_t pbVmreadBitmap = (uint8_t )pVM->hmr0.s.vmx.pvVmreadBitmap;
2152	uint8_t pbVmwriteBitmap = (uint8_t )pVM->hmr0.s.vmx.pvVmwriteBitmap;
2153	ASMMemFill32(pbVmreadBitmap, cbBitmap, UINT32_C(0xffffffff));
2154	ASMMemFill32(pbVmwriteBitmap, cbBitmap, UINT32_C(0xffffffff));
2155
2156	/*
2157	* Skip intercepting VMREAD/VMWRITE to guest read/write fields in the
2158	* VMREAD and VMWRITE bitmaps.
2159	*/
2160	{
2161	uint32_t const *paShadowVmcsFields = pVM->hmr0.s.vmx.paShadowVmcsFields;
2162	uint32_t const cShadowVmcsFields = pVM->hmr0.s.vmx.cShadowVmcsFields;
2163	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
2164	{
2165	uint32_t const uVmcsField = paShadowVmcsFields[i];
2166	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
2167	Assert(uVmcsField >> 3 < cbBitmap);
2168	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
2169	ASMBitClear(pbVmwriteBitmap + (uVmcsField >> 3), uVmcsField & 7);
2170	}
2171	}
2172
2173	/*
2174	* Skip intercepting VMREAD for guest read-only fields in the VMREAD bitmap
2175	* if the host supports VMWRITE to all supported VMCS fields.
2176	*/
2177	if (g_HmMsrs.u.vmx.u64Misc & VMX_MISC_VMWRITE_ALL)
2178	{
2179	uint32_t const *paShadowVmcsRoFields = pVM->hmr0.s.vmx.paShadowVmcsRoFields;
2180	uint32_t const cShadowVmcsRoFields = pVM->hmr0.s.vmx.cShadowVmcsRoFields;
2181	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
2182	{
2183	uint32_t const uVmcsField = paShadowVmcsRoFields[i];
2184	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
2185	Assert(uVmcsField >> 3 < cbBitmap);
2186	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
2187	}
2188	}
2189	}
2190	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
2191
2192
2193	/**
2194	* Sets up the APIC-access page address for the VMCS.
2195	*
2196	* @param pVCpu The cross context virtual CPU structure.
2197	*/
2198	DECLINLINE(void) vmxHCSetupVmcsApicAccessAddr(PVMCPUCC pVCpu)
2199	{
2200	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.HCPhysApicAccess;
2201	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
2202	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2203	int rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
2204	AssertRC(rc);
2205	}
2206
2207	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2208
2209	/**
2210	* Sets up the VMREAD bitmap address for the VMCS.
2211	*
2212	* @param pVCpu The cross context virtual CPU structure.
2213	*/
2214	DECLINLINE(void) vmxHCSetupVmcsVmreadBitmapAddr(PVMCPUCC pVCpu)
2215	{
2216	RTHCPHYS const HCPhysVmreadBitmap = pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.HCPhysVmreadBitmap;
2217	Assert(HCPhysVmreadBitmap != NIL_RTHCPHYS);
2218	Assert(!(HCPhysVmreadBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2219	int rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL, HCPhysVmreadBitmap);
2220	AssertRC(rc);
2221	}
2222
2223
2224	/**
2225	* Sets up the VMWRITE bitmap address for the VMCS.
2226	*
2227	* @param pVCpu The cross context virtual CPU structure.
2228	*/
2229	DECLINLINE(void) vmxHCSetupVmcsVmwriteBitmapAddr(PVMCPUCC pVCpu)
2230	{
2231	RTHCPHYS const HCPhysVmwriteBitmap = pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.HCPhysVmwriteBitmap;
2232	Assert(HCPhysVmwriteBitmap != NIL_RTHCPHYS);
2233	Assert(!(HCPhysVmwriteBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2234	int rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL, HCPhysVmwriteBitmap);
2235	AssertRC(rc);
2236	}
2237
2238	#endif
2239
2240	/**
2241	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
2242	*
2243	* @param pVCpu The cross context virtual CPU structure.
2244	* @param pVmcsInfo The VMCS info. object.
2245	*/
2246	static void vmxHCSetupVmcsMsrPermissions(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2247	{
2248	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
2249
2250	/*
2251	* By default, ensure guest attempts to access any MSR cause VM-exits.
2252	* This shall later be relaxed for specific MSRs as necessary.
2253	*
2254	* Note: For nested-guests, the entire bitmap will be merged prior to
2255	* executing the nested-guest using hardware-assisted VMX and hence there
2256	* is no need to perform this operation. See vmxHCMergeMsrBitmapNested.
2257	*/
2258	Assert(pVmcsInfo->pvMsrBitmap);
2259	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
2260
2261	/*
2262	* The guest can access the following MSRs (read, write) without causing
2263	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
2264	*/
2265	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2266	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
2267	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
2268	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
2269	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2270	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
2271
2272	/*
2273	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
2274	* associated with then. We never need to intercept access (writes need to be
2275	* executed without causing a VM-exit, reads will #GP fault anyway).
2276	*
2277	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
2278	* read/write them. We swap the guest/host MSR value using the
2279	* auto-load/store MSR area.
2280	*/
2281	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2282	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
2283	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
2284	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
2285	if (pVM->cpum.ro.GuestFeatures.fIbrs)
2286	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
2287
2288	/*
2289	* Allow full read/write access for the following MSRs (mandatory for VT-x)
2290	* required for 64-bit guests.
2291	*/
2292	if (pVM->hmr0.s.fAllow64BitGuests)
2293	{
2294	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
2295	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
2296	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
2297	vmxHCSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2298	}
2299
2300	/*
2301	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
2302	*/
2303	#ifdef VBOX_STRICT
2304	Assert(pVmcsInfo->pvMsrBitmap);
2305	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
2306	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
2307	#endif
2308	}
2309
2310
2311	/**
2312	* Sets up pin-based VM-execution controls in the VMCS.
2313	*
2314	* @returns VBox status code.
2315	* @param pVCpu The cross context virtual CPU structure.
2316	* @param pVmcsInfo The VMCS info. object.
2317	*/
2318	static int vmxHCSetupVmcsPinCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2319	{
2320	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2321	uint32_t fVal = g_HmMsrs.u.vmx.PinCtls.n.allowed0; /* Bits set here must always be set. */
2322	uint32_t const fZap = g_HmMsrs.u.vmx.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2323
2324	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2325	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2326
2327	if (g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2328	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2329
2330	/* Enable the VMX-preemption timer. */
2331	if (VM_IS_VMX_PREEMPT_TIMER_USED(pVM))
2332	{
2333	Assert(g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2334	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
2335	}
2336
2337	#if 0
2338	/* Enable posted-interrupt processing. */
2339	if (pVM->hm.s.fPostedIntrs)
2340	{
2341	Assert(g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2342	Assert(g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2343	fVal \|= VMX_PIN_CTLS_POSTED_INT;
2344	}
2345	#endif
2346
2347	if ((fVal & fZap) != fVal)
2348	{
2349	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2350	g_HmMsrs.u.vmx.PinCtls.n.allowed0, fVal, fZap));
2351	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2352	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2353	}
2354
2355	/* Commit it to the VMCS and update our cache. */
2356	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2357	AssertRC(rc);
2358	pVmcsInfo->u32PinCtls = fVal;
2359
2360	return VINF_SUCCESS;
2361	}
2362
2363
2364	/**
2365	* Sets up secondary processor-based VM-execution controls in the VMCS.
2366	*
2367	* @returns VBox status code.
2368	* @param pVCpu The cross context virtual CPU structure.
2369	* @param pVmcsInfo The VMCS info. object.
2370	*/
2371	static int vmxHCSetupVmcsProcCtls2(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2372	{
2373	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2374	uint32_t fVal = g_HmMsrs.u.vmx.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
2375	uint32_t const fZap = g_HmMsrs.u.vmx.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2376
2377	/* WBINVD causes a VM-exit. */
2378	if (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2379	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
2380
2381	/* Enable EPT (aka nested-paging). */
2382	if (VM_IS_VMX_NESTED_PAGING(pVM))
2383	fVal \|= VMX_PROC_CTLS2_EPT;
2384
2385	/* Enable the INVPCID instruction if we expose it to the guest and is supported
2386	by the hardware. Without this, guest executing INVPCID would cause a #UD. */
2387	if ( pVM->cpum.ro.GuestFeatures.fInvpcid
2388	&& (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID))
2389	fVal \|= VMX_PROC_CTLS2_INVPCID;
2390
2391	/* Enable VPID. */
2392	if (pVM->hmr0.s.vmx.fVpid)
2393	fVal \|= VMX_PROC_CTLS2_VPID;
2394
2395	/* Enable unrestricted guest execution. */
2396	if (VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
2397	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
2398
2399	#if 0
2400	if (pVM->hm.s.fVirtApicRegs)
2401	{
2402	/* Enable APIC-register virtualization. */
2403	Assert(g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
2404	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
2405
2406	/* Enable virtual-interrupt delivery. */
2407	Assert(g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
2408	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
2409	}
2410	#endif
2411
2412	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is
2413	where the TPR shadow resides. */
2414	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2415	* done dynamically. */
2416	if (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
2417	{
2418	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
2419	vmxHCSetupVmcsApicAccessAddr(pVCpu);
2420	}
2421
2422	/* Enable the RDTSCP instruction if we expose it to the guest and is supported
2423	by the hardware. Without this, guest executing RDTSCP would cause a #UD. */
2424	if ( pVM->cpum.ro.GuestFeatures.fRdTscP
2425	&& (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP))
2426	fVal \|= VMX_PROC_CTLS2_RDTSCP;
2427
2428	/* Enable Pause-Loop exiting. */
2429	if ( (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
2430	&& pVM->hm.s.vmx.cPleGapTicks
2431	&& pVM->hm.s.vmx.cPleWindowTicks)
2432	{
2433	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
2434
2435	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks); AssertRC(rc);
2436	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks); AssertRC(rc);
2437	}
2438
2439	if ((fVal & fZap) != fVal)
2440	{
2441	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2442	g_HmMsrs.u.vmx.ProcCtls2.n.allowed0, fVal, fZap));
2443	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2444	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2445	}
2446
2447	/* Commit it to the VMCS and update our cache. */
2448	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2449	AssertRC(rc);
2450	pVmcsInfo->u32ProcCtls2 = fVal;
2451
2452	return VINF_SUCCESS;
2453	}
2454
2455
2456	/**
2457	* Sets up processor-based VM-execution controls in the VMCS.
2458	*
2459	* @returns VBox status code.
2460	* @param pVCpu The cross context virtual CPU structure.
2461	* @param pVmcsInfo The VMCS info. object.
2462	*/
2463	static int vmxHCSetupVmcsProcCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2464	{
2465	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2466	uint32_t fVal = g_HmMsrs.u.vmx.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
2467	uint32_t const fZap = g_HmMsrs.u.vmx.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2468
2469	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
2470	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2471	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2472	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2473	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2474	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2475	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2476
2477	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2478	if ( !(g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
2479	\|\| (g_HmMsrs.u.vmx.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
2480	{
2481	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2482	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2483	}
2484
2485	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2486	if (!VM_IS_VMX_NESTED_PAGING(pVM))
2487	{
2488	Assert(!VM_IS_VMX_UNRESTRICTED_GUEST(pVM));
2489	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
2490	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
2491	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
2492	}
2493
2494	#ifdef IN_INRG0
2495	/* Use TPR shadowing if supported by the CPU. */
2496	if ( PDMHasApic(pVM)
2497	&& (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
2498	{
2499	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2500	/* CR8 writes cause a VM-exit based on TPR threshold. */
2501	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
2502	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
2503	vmxHCSetupVmcsVirtApicAddr(pVmcsInfo);
2504	}
2505	else
2506	{
2507	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
2508	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
2509	if (pVM->hmr0.s.fAllow64BitGuests)
2510	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2511	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2512	}
2513
2514	/* Use MSR-bitmaps if supported by the CPU. */
2515	if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2516	{
2517	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
2518	vmxHCSetupVmcsMsrBitmapAddr(pVmcsInfo);
2519	}
2520	#endif
2521
2522	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2523	if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2524	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
2525
2526	if ((fVal & fZap) != fVal)
2527	{
2528	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2529	g_HmMsrs.u.vmx.ProcCtls.n.allowed0, fVal, fZap));
2530	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2531	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2532	}
2533
2534	/* Commit it to the VMCS and update our cache. */
2535	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2536	AssertRC(rc);
2537	pVmcsInfo->u32ProcCtls = fVal;
2538
2539	/* Set up MSR permissions that don't change through the lifetime of the VM. */
2540	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2541	vmxHCSetupVmcsMsrPermissions(pVCpu, pVmcsInfo);
2542
2543	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2544	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2545	return vmxHCSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
2546
2547	/* Sanity check, should not really happen. */
2548	if (RT_LIKELY(!VM_IS_VMX_UNRESTRICTED_GUEST(pVM)))
2549	{ /* likely */ }
2550	else
2551	{
2552	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_INVALID_UX_COMBO;
2553	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2554	}
2555
2556	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2557	return VINF_SUCCESS;
2558	}
2559
2560
2561	/**
2562	* Sets up miscellaneous (everything other than Pin, Processor and secondary
2563	* Processor-based VM-execution) control fields in the VMCS.
2564	*
2565	* @returns VBox status code.
2566	* @param pVCpu The cross context virtual CPU structure.
2567	* @param pVmcsInfo The VMCS info. object.
2568	*/
2569	static int vmxHCSetupVmcsMiscCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2570	{
2571	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2572	if (pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fUseVmcsShadowing)
2573	{
2574	vmxHCSetupVmcsVmreadBitmapAddr(pVCpu);
2575	vmxHCSetupVmcsVmwriteBitmapAddr(pVCpu);
2576	}
2577	#endif
2578
2579	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
2580	int rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
2581	AssertRC(rc);
2582
2583	rc = vmxHCSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
2584	if (RT_SUCCESS(rc))
2585	{
2586	uint64_t const u64Cr0Mask = vmxHCGetFixedCr0Mask(pVCpu);
2587	uint64_t const u64Cr4Mask = vmxHCGetFixedCr4Mask(pVCpu);
2588
2589	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask); AssertRC(rc);
2590	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask); AssertRC(rc);
2591
2592	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
2593	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
2594
2595	if (pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fLbr)
2596	{
2597	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS64_GUEST_DEBUGCTL_FULL, MSR_IA32_DEBUGCTL_LBR);
2598	AssertRC(rc);
2599	}
2600	return VINF_SUCCESS;
2601	}
2602	else
2603	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
2604	return rc;
2605	}
2606
2607
2608	/**
2609	* Sets up the initial exception bitmap in the VMCS based on static conditions.
2610	*
2611	* We shall setup those exception intercepts that don't change during the
2612	* lifetime of the VM here. The rest are done dynamically while loading the
2613	* guest state.
2614	*
2615	* @param pVCpu The cross context virtual CPU structure.
2616	* @param pVmcsInfo The VMCS info. object.
2617	*/
2618	static void vmxHCSetupVmcsXcptBitmap(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2619	{
2620	/*
2621	* The following exceptions are always intercepted:
2622	*
2623	* #AC - To prevent the guest from hanging the CPU and for dealing with
2624	* split-lock detecting host configs.
2625	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
2626	* recursive #DBs can cause a CPU hang.
2627	* #PF - To sync our shadow page tables when nested-paging is not used.
2628	*/
2629	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging;
2630	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
2631	\| RT_BIT(X86_XCPT_DB)
2632	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
2633
2634	/* Commit it to the VMCS. */
2635	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2636	AssertRC(rc);
2637
2638	/* Update our cache of the exception bitmap. */
2639	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
2640	}
2641
2642
2643	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2644	/**
2645	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
2646	*
2647	* @returns VBox status code.
2648	* @param pVmcsInfo The VMCS info. object.
2649	*/
2650	static int vmxHCSetupVmcsCtlsNested(PVMXVMCSINFO pVmcsInfo)
2651	{
2652	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
2653	int rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
2654	AssertRC(rc);
2655
2656	rc = vmxHCSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
2657	if (RT_SUCCESS(rc))
2658	{
2659	if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2660	vmxHCSetupVmcsMsrBitmapAddr(pVmcsInfo);
2661
2662	/* Paranoia - We've not yet initialized these, they shall be done while merging the VMCS. */
2663	Assert(!pVmcsInfo->u64Cr0Mask);
2664	Assert(!pVmcsInfo->u64Cr4Mask);
2665	return VINF_SUCCESS;
2666	}
2667	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
2668	return rc;
2669	}
2670	#endif
2671	#endif /* !IN_RING0 */
2672
2673
2674	/**
2675	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
2676	* VMCS.
2677	*
2678	* This is typically required when the guest changes paging mode.
2679	*
2680	* @returns VBox status code.
2681	* @param pVCpu The cross context virtual CPU structure.
2682	* @param pVmxTransient The VMX-transient structure.
2683	*
2684	* @remarks Requires EFER.
2685	* @remarks No-long-jump zone!!!
2686	*/
2687	static int vmxHCExportGuestEntryExitCtls(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
2688	{
2689	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
2690	{
2691	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2692	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2693
2694	/*
2695	* VM-entry controls.
2696	*/
2697	{
2698	uint32_t fVal = g_HmMsrs.u.vmx.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
2699	uint32_t const fZap = g_HmMsrs.u.vmx.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2700
2701	/*
2702	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
2703	* The first VT-x capable CPUs only supported the 1-setting of this bit.
2704	*
2705	* For nested-guests, this is a mandatory VM-entry control. It's also
2706	* required because we do not want to leak host bits to the nested-guest.
2707	*/
2708	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
2709
2710	/*
2711	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
2712	*
2713	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
2714	* required to get the nested-guest working with hardware-assisted VMX execution.
2715	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested hypervisor
2716	* can skip intercepting changes to the EFER MSR. This is why it needs to be done
2717	* here rather than while merging the guest VMCS controls.
2718	*/
2719	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
2720	{
2721	Assert(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME);
2722	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
2723	}
2724	else
2725	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
2726
2727	/*
2728	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
2729	*
2730	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
2731	* regardless of whether the nested-guest VMCS specifies it because we are free to
2732	* load whatever MSRs we require and we do not need to modify the guest visible copy
2733	* of the VM-entry MSR load area.
2734	*/
2735	if ( g_fHmVmxSupportsVmcsEfer
2736	&& vmxHCShouldSwapEferMsr(pVCpu, pVmxTransient))
2737	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
2738	else
2739	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
2740
2741	/*
2742	* The following should -not- be set (since we're not in SMM mode):
2743	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
2744	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
2745	*/
2746
2747	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
2748	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
2749
2750	if ((fVal & fZap) == fVal)
2751	{ /* likely */ }
2752	else
2753	{
2754	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2755	g_HmMsrs.u.vmx.EntryCtls.n.allowed0, fVal, fZap));
2756	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_CTRL_ENTRY;
2757	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2758	}
2759
2760	/* Commit it to the VMCS. */
2761	if (pVmcsInfo->u32EntryCtls != fVal)
2762	{
2763	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY, fVal);
2764	AssertRC(rc);
2765	pVmcsInfo->u32EntryCtls = fVal;
2766	}
2767	}
2768
2769	/*
2770	* VM-exit controls.
2771	*/
2772	{
2773	uint32_t fVal = g_HmMsrs.u.vmx.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
2774	uint32_t const fZap = g_HmMsrs.u.vmx.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2775
2776	/*
2777	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
2778	* supported the 1-setting of this bit.
2779	*
2780	* For nested-guests, we set the "save debug controls" as the converse
2781	* "load debug controls" is mandatory for nested-guests anyway.
2782	*/
2783	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
2784
2785	/*
2786	* Set the host long mode active (EFER.LMA) bit (which Intel calls
2787	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
2788	* host EFER.LMA and EFER.LME bit to this value. See assertion in
2789	* vmxHCExportHostMsrs().
2790	*
2791	* For nested-guests, we always set this bit as we do not support 32-bit
2792	* hosts.
2793	*/
2794	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
2795
2796	#ifdef IN_RING0
2797	/*
2798	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
2799	*
2800	* For nested-guests, we should use the "save IA32_EFER" control if we also
2801	* used the "load IA32_EFER" control while exporting VM-entry controls.
2802	*/
2803	if ( g_fHmVmxSupportsVmcsEfer
2804	&& vmxHCShouldSwapEferMsr(pVCpu, pVmxTransient))
2805	{
2806	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
2807	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
2808	}
2809	#endif
2810
2811	/*
2812	* Enable saving of the VMX-preemption timer value on VM-exit.
2813	* For nested-guests, currently not exposed/used.
2814	*/
2815	/** @todo r=bird: Measure performance hit because of this vs. always rewriting
2816	* the timer value. */
2817	if (VM_IS_VMX_PREEMPT_TIMER_USED(pVM))
2818	{
2819	Assert(g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER);
2820	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
2821	}
2822
2823	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2824	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
2825
2826	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
2827	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
2828	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
2829
2830	if ((fVal & fZap) == fVal)
2831	{ /* likely */ }
2832	else
2833	{
2834	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2835	g_HmMsrs.u.vmx.ExitCtls.n.allowed0, fVal, fZap));
2836	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_CTRL_EXIT;
2837	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2838	}
2839
2840	/* Commit it to the VMCS. */
2841	if (pVmcsInfo->u32ExitCtls != fVal)
2842	{
2843	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXIT, fVal);
2844	AssertRC(rc);
2845	pVmcsInfo->u32ExitCtls = fVal;
2846	}
2847	}
2848
2849	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
2850	}
2851	return VINF_SUCCESS;
2852	}
2853
2854
2855	/**
2856	* Sets the TPR threshold in the VMCS.
2857	*
2858	* @param pVCpu The cross context virtual CPU structure.
2859	* @param pVmcsInfo The VMCS info. object.
2860	* @param u32TprThreshold The TPR threshold (task-priority class only).
2861	*/
2862	DECLINLINE(void) vmxHCApicSetTprThreshold(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
2863	{
2864	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
2865	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
2866	RT_NOREF(pVmcsInfo);
2867	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2868	AssertRC(rc);
2869	}
2870
2871
2872	/**
2873	* Exports the guest APIC TPR state into the VMCS.
2874	*
2875	* @param pVCpu The cross context virtual CPU structure.
2876	* @param pVmxTransient The VMX-transient structure.
2877	*
2878	* @remarks No-long-jump zone!!!
2879	*/
2880	static void vmxHCExportGuestApicTpr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
2881	{
2882	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
2883	{
2884	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
2885
2886	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2887	if (!pVmxTransient->fIsNestedGuest)
2888	{
2889	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
2890	&& APICIsEnabled(pVCpu))
2891	{
2892	/*
2893	* Setup TPR shadowing.
2894	*/
2895	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
2896	{
2897	bool fPendingIntr = false;
2898	uint8_t u8Tpr = 0;
2899	uint8_t u8PendingIntr = 0;
2900	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2901	AssertRC(rc);
2902
2903	/*
2904	* If there are interrupts pending but masked by the TPR, instruct VT-x to
2905	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
2906	* priority of the pending interrupt so we can deliver the interrupt. If there
2907	* are no interrupts pending, set threshold to 0 to not cause any
2908	* TPR-below-threshold VM-exits.
2909	*/
2910	uint32_t u32TprThreshold = 0;
2911	if (fPendingIntr)
2912	{
2913	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
2914	(which is the Task-Priority Class). */
2915	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
2916	const uint8_t u8TprPriority = u8Tpr >> 4;
2917	if (u8PendingPriority <= u8TprPriority)
2918	u32TprThreshold = u8PendingPriority;
2919	}
2920
2921	vmxHCApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
2922	}
2923	}
2924	}
2925	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
2926	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
2927	}
2928	}
2929
2930
2931	/**
2932	* Gets the guest interruptibility-state and updates related force-flags.
2933	*
2934	* @returns Guest's interruptibility-state.
2935	* @param pVCpu The cross context virtual CPU structure.
2936	*
2937	* @remarks No-long-jump zone!!!
2938	*/
2939	static uint32_t vmxHCGetGuestIntrStateAndUpdateFFs(PVMCPUCC pVCpu)
2940	{
2941	/*
2942	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
2943	*/
2944	uint32_t fIntrState = 0;
2945	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2946	{
2947	/* If inhibition is active, RIP and RFLAGS should've been imported from the VMCS already. */
2948	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
2949
2950	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2951	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
2952	{
2953	if (pCtx->eflags.Bits.u1IF)
2954	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
2955	else
2956	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
2957	}
2958	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2959	{
2960	/*
2961	* We can clear the inhibit force flag as even if we go back to the recompiler
2962	* without executing guest code in VT-x, the flag's condition to be cleared is
2963	* met and thus the cleared state is correct.
2964	*/
2965	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2966	}
2967	}
2968
2969	/*
2970	* Check if we should inhibit NMI delivery.
2971	*/
2972	if (CPUMIsGuestNmiBlocking(pVCpu))
2973	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
2974
2975	/*
2976	* Validate.
2977	*/
2978	#ifdef VBOX_STRICT
2979	/* We don't support block-by-SMI yet.*/
2980	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI));
2981
2982	/* Block-by-STI must not be set when interrupts are disabled. */
2983	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
2984	{
2985	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
2986	Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_IF);
2987	}
2988	#endif
2989
2990	return fIntrState;
2991	}
2992
2993
2994	/**
2995	* Exports the exception intercepts required for guest execution in the VMCS.
2996	*
2997	* @param pVCpu The cross context virtual CPU structure.
2998	* @param pVmxTransient The VMX-transient structure.
2999	*
3000	* @remarks No-long-jump zone!!!
3001	*/
3002	static void vmxHCExportGuestXcptIntercepts(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
3003	{
3004	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
3005	{
3006	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
3007	if ( !pVmxTransient->fIsNestedGuest
3008	&& VCPU_2_VMXSTATE(pVCpu).fGIMTrapXcptUD)
3009	vmxHCAddXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
3010	else
3011	vmxHCRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
3012
3013	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
3014	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
3015	}
3016	}
3017
3018
3019	/**
3020	* Exports the guest's RIP into the guest-state area in the VMCS.
3021	*
3022	* @param pVCpu The cross context virtual CPU structure.
3023	*
3024	* @remarks No-long-jump zone!!!
3025	*/
3026	static void vmxHCExportGuestRip(PVMCPUCC pVCpu)
3027	{
3028	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_RIP)
3029	{
3030	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
3031
3032	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
3033	AssertRC(rc);
3034
3035	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3036	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
3037	}
3038	}
3039
3040
3041	#ifdef IN_RING0
3042	/**
3043	* Exports the guest's RSP into the guest-state area in the VMCS.
3044	*
3045	* @param pVCpu The cross context virtual CPU structure.
3046	*
3047	* @remarks No-long-jump zone!!!
3048	*/
3049	static void vmxHCExportGuestRsp(PVMCPUCC pVCpu)
3050	{
3051	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_RSP)
3052	{
3053	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
3054
3055	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
3056	AssertRC(rc);
3057
3058	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3059	Log4Func(("rsp=%#RX64\n", pVCpu->cpum.GstCtx.rsp));
3060	}
3061	}
3062	#endif
3063
3064
3065	/**
3066	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
3067	*
3068	* @param pVCpu The cross context virtual CPU structure.
3069	* @param pVmxTransient The VMX-transient structure.
3070	*
3071	* @remarks No-long-jump zone!!!
3072	*/
3073	static void vmxHCExportGuestRflags(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
3074	{
3075	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3076	{
3077	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
3078
3079	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3080	Let us assert it as such and use 32-bit VMWRITE. */
3081	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
3082	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
3083	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3084	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
3085
3086	#ifdef IN_RING0
3087	/*
3088	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3089	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3090	* can run the real-mode guest code under Virtual 8086 mode.
3091	*/
3092	PVMXVMCSINFOSHARED pVmcsInfo = pVmxTransient->pVmcsInfo->pShared;
3093	if (pVmcsInfo->RealMode.fRealOnV86Active)
3094	{
3095	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3096	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3097	Assert(!pVmxTransient->fIsNestedGuest);
3098	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3099	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3100	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3101	}
3102	#else
3103	RT_NOREF(pVmxTransient);
3104	#endif
3105
3106	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3107	AssertRC(rc);
3108
3109	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3110	Log4Func(("eflags=%#RX32\n", fEFlags.u32));
3111	}
3112	}
3113
3114
3115	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3116	/**
3117	* Copies the nested-guest VMCS to the shadow VMCS.
3118	*
3119	* @returns VBox status code.
3120	* @param pVCpu The cross context virtual CPU structure.
3121	* @param pVmcsInfo The VMCS info. object.
3122	*
3123	* @remarks No-long-jump zone!!!
3124	*/
3125	static int vmxHCCopyNstGstToShadowVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3126	{
3127	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
3128	PCVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
3129
3130	/*
3131	* Disable interrupts so we don't get preempted while the shadow VMCS is the
3132	* current VMCS, as we may try saving guest lazy MSRs.
3133	*
3134	* Strictly speaking the lazy MSRs are not in the VMCS, but I'd rather not risk
3135	* calling the import VMCS code which is currently performing the guest MSR reads
3136	* (on 64-bit hosts) and accessing the auto-load/store MSR area on 32-bit hosts
3137	* and the rest of the VMX leave session machinery.
3138	*/
3139	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
3140
3141	int rc = vmxHCLoadShadowVmcs(pVmcsInfo);
3142	if (RT_SUCCESS(rc))
3143	{
3144	/*
3145	* Copy all guest read/write VMCS fields.
3146	*
3147	* We don't check for VMWRITE failures here for performance reasons and
3148	* because they are not expected to fail, barring irrecoverable conditions
3149	* like hardware errors.
3150	*/
3151	uint32_t const cShadowVmcsFields = pVM->hmr0.s.vmx.cShadowVmcsFields;
3152	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
3153	{
3154	uint64_t u64Val;
3155	uint32_t const uVmcsField = pVM->hmr0.s.vmx.paShadowVmcsFields[i];
3156	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
3157	VMX_VMCS_WRITE_64(pVCpu, uVmcsField, u64Val);
3158	}
3159
3160	/*
3161	* If the host CPU supports writing all VMCS fields, copy the guest read-only
3162	* VMCS fields, so the guest can VMREAD them without causing a VM-exit.
3163	*/
3164	if (g_HmMsrs.u.vmx.u64Misc & VMX_MISC_VMWRITE_ALL)
3165	{
3166	uint32_t const cShadowVmcsRoFields = pVM->hmr0.s.vmx.cShadowVmcsRoFields;
3167	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
3168	{
3169	uint64_t u64Val;
3170	uint32_t const uVmcsField = pVM->hmr0.s.vmx.paShadowVmcsRoFields[i];
3171	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
3172	VMX_VMCS_WRITE_64(pVCpu, uVmcsField, u64Val);
3173	}
3174	}
3175
3176	rc = vmxHCClearShadowVmcs(pVmcsInfo);
3177	rc \|= vmxHCLoadVmcs(pVmcsInfo);
3178	}
3179
3180	ASMSetFlags(fEFlags);
3181	return rc;
3182	}
3183
3184
3185	/**
3186	* Copies the shadow VMCS to the nested-guest VMCS.
3187	*
3188	* @returns VBox status code.
3189	* @param pVCpu The cross context virtual CPU structure.
3190	* @param pVmcsInfo The VMCS info. object.
3191	*
3192	* @remarks Called with interrupts disabled.
3193	*/
3194	static int vmxHCCopyShadowToNstGstVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3195	{
3196	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3197	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
3198	PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
3199
3200	int rc = vmxHCLoadShadowVmcs(pVmcsInfo);
3201	if (RT_SUCCESS(rc))
3202	{
3203	/*
3204	* Copy guest read/write fields from the shadow VMCS.
3205	* Guest read-only fields cannot be modified, so no need to copy them.
3206	*
3207	* We don't check for VMREAD failures here for performance reasons and
3208	* because they are not expected to fail, barring irrecoverable conditions
3209	* like hardware errors.
3210	*/
3211	uint32_t const cShadowVmcsFields = pVM->hmr0.s.vmx.cShadowVmcsFields;
3212	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
3213	{
3214	uint64_t u64Val;
3215	uint32_t const uVmcsField = pVM->hmr0.s.vmx.paShadowVmcsFields[i];
3216	VMX_VMCS_READ_64(pVCpu, uVmcsField, &u64Val);
3217	IEMWriteVmxVmcsField(pVmcsNstGst, uVmcsField, u64Val);
3218	}
3219
3220	rc = vmxHCClearShadowVmcs(pVmcsInfo);
3221	rc \|= vmxHCLoadVmcs(pVmcsInfo);
3222	}
3223	return rc;
3224	}
3225
3226
3227	/**
3228	* Enables VMCS shadowing for the given VMCS info. object.
3229	*
3230	* @param pVmcsInfo The VMCS info. object.
3231	*
3232	* @remarks No-long-jump zone!!!
3233	*/
3234	static void vmxHCEnableVmcsShadowing(PVMXVMCSINFO pVmcsInfo)
3235	{
3236	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
3237	if (!(uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING))
3238	{
3239	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
3240	uProcCtls2 \|= VMX_PROC_CTLS2_VMCS_SHADOWING;
3241	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc);
3242	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, pVmcsInfo->HCPhysShadowVmcs); AssertRC(rc);
3243	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
3244	pVmcsInfo->u64VmcsLinkPtr = pVmcsInfo->HCPhysShadowVmcs;
3245	Log4Func(("Enabled\n"));
3246	}
3247	}
3248
3249
3250	/**
3251	* Disables VMCS shadowing for the given VMCS info. object.
3252	*
3253	* @param pVmcsInfo The VMCS info. object.
3254	*
3255	* @remarks No-long-jump zone!!!
3256	*/
3257	static void vmxHCDisableVmcsShadowing(PVMXVMCSINFO pVmcsInfo)
3258	{
3259	/*
3260	* We want all VMREAD and VMWRITE instructions to cause VM-exits, so we clear the
3261	* VMCS shadowing control. However, VM-entry requires the shadow VMCS indicator bit
3262	* to match the VMCS shadowing control if the VMCS link pointer is not NIL_RTHCPHYS.
3263	* Hence, we must also reset the VMCS link pointer to ensure VM-entry does not fail.
3264	*
3265	* See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
3266	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
3267	*/
3268	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
3269	if (uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
3270	{
3271	uProcCtls2 &= ~VMX_PROC_CTLS2_VMCS_SHADOWING;
3272	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc);
3273	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS); AssertRC(rc);
3274	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
3275	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
3276	Log4Func(("Disabled\n"));
3277	}
3278	}
3279	#endif
3280
3281
3282	#ifdef IN_RING0
3283	/**
3284	* Exports the guest hardware-virtualization state.
3285	*
3286	* @returns VBox status code.
3287	* @param pVCpu The cross context virtual CPU structure.
3288	* @param pVmxTransient The VMX-transient structure.
3289	*
3290	* @remarks No-long-jump zone!!!
3291	*/
3292	static int vmxHCExportGuestHwvirtState(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
3293	{
3294	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_HWVIRT)
3295	{
3296	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3297	/*
3298	* Check if the VMX feature is exposed to the guest and if the host CPU supports
3299	* VMCS shadowing.
3300	*/
3301	if (pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fUseVmcsShadowing)
3302	{
3303	/*
3304	* If the nested hypervisor has loaded a current VMCS and is in VMX root mode,
3305	* copy the nested hypervisor's current VMCS into the shadow VMCS and enable
3306	* VMCS shadowing to skip intercepting some or all VMREAD/VMWRITE VM-exits.
3307	*
3308	* We check for VMX root mode here in case the guest executes VMXOFF without
3309	* clearing the current VMCS pointer and our VMXOFF instruction emulation does
3310	* not clear the current VMCS pointer.
3311	*/
3312	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
3313	if ( CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx)
3314	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
3315	&& CPUMIsGuestVmxCurrentVmcsValid(&pVCpu->cpum.GstCtx))
3316	{
3317	/* Paranoia. */
3318	Assert(!pVmxTransient->fIsNestedGuest);
3319
3320	/*
3321	* For performance reasons, also check if the nested hypervisor's current VMCS
3322	* was newly loaded or modified before copying it to the shadow VMCS.
3323	*/
3324	if (!VCPU_2_VMXSTATE(pVCpu).vmx.fCopiedNstGstToShadowVmcs)
3325	{
3326	int rc = vmxHCCopyNstGstToShadowVmcs(pVCpu, pVmcsInfo);
3327	AssertRCReturn(rc, rc);
3328	VCPU_2_VMXSTATE(pVCpu).vmx.fCopiedNstGstToShadowVmcs = true;
3329	}
3330	vmxHCEnableVmcsShadowing(pVmcsInfo);
3331	}
3332	else
3333	vmxHCDisableVmcsShadowing(pVmcsInfo);
3334	}
3335	#else
3336	NOREF(pVmxTransient);
3337	#endif
3338	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_HWVIRT);
3339	}
3340	return VINF_SUCCESS;
3341	}
3342	#endif
3343
3344
3345	/**
3346	* Exports the guest CR0 control register into the guest-state area in the VMCS.
3347	*
3348	* The guest FPU state is always pre-loaded hence we don't need to bother about
3349	* sharing FPU related CR0 bits between the guest and host.
3350	*
3351	* @returns VBox status code.
3352	* @param pVCpu The cross context virtual CPU structure.
3353	* @param pVmxTransient The VMX-transient structure.
3354	*
3355	* @remarks No-long-jump zone!!!
3356	*/
3357	static int vmxHCExportGuestCR0(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
3358	{
3359	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CR0)
3360	{
3361	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3362	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
3363
3364	uint64_t fSetCr0 = g_HmMsrs.u.vmx.u64Cr0Fixed0;
3365	uint64_t const fZapCr0 = g_HmMsrs.u.vmx.u64Cr0Fixed1;
3366	if (VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
3367	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
3368	else
3369	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3370
3371	if (!pVmxTransient->fIsNestedGuest)
3372	{
3373	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3374	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
3375	uint64_t const u64ShadowCr0 = u64GuestCr0;
3376	Assert(!RT_HI_U32(u64GuestCr0));
3377
3378	/*
3379	* Setup VT-x's view of the guest CR0.
3380	*/
3381	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
3382	if (VM_IS_VMX_NESTED_PAGING(pVM))
3383	{
3384	if (CPUMIsGuestPagingEnabled(pVCpu))
3385	{
3386	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3387	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
3388	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
3389	}
3390	else
3391	{
3392	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3393	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
3394	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3395	}
3396
3397	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3398	if (VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
3399	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
3400	}
3401	else
3402	{
3403	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3404	u64GuestCr0 \|= X86_CR0_WP;
3405	}
3406
3407	/*
3408	* Guest FPU bits.
3409	*
3410	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3411	* using CR0.TS.
3412	*
3413	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3414	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3415	*/
3416	u64GuestCr0 \|= X86_CR0_NE;
3417
3418	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3419	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
3420
3421	/*
3422	* Update exception intercepts.
3423	*/
3424	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
3425	#ifdef IN_RING0
3426	if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
3427	{
3428	Assert(PDMVmmDevHeapIsEnabled(pVM));
3429	Assert(pVM->hm.s.vmx.pRealModeTSS);
3430	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3431	}
3432	else
3433	#endif
3434	{
3435	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3436	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3437	if (fInterceptMF)
3438	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
3439	}
3440
3441	/* Additional intercepts for debugging, define these yourself explicitly. */
3442	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3443	uXcptBitmap \|= 0
3444	\| RT_BIT(X86_XCPT_BP)
3445	\| RT_BIT(X86_XCPT_DE)
3446	\| RT_BIT(X86_XCPT_NM)
3447	\| RT_BIT(X86_XCPT_TS)
3448	\| RT_BIT(X86_XCPT_UD)
3449	\| RT_BIT(X86_XCPT_NP)
3450	\| RT_BIT(X86_XCPT_SS)
3451	\| RT_BIT(X86_XCPT_GP)
3452	\| RT_BIT(X86_XCPT_PF)
3453	\| RT_BIT(X86_XCPT_MF)
3454	;
3455	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3456	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
3457	#endif
3458	if (VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv)
3459	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
3460	Assert(VM_IS_VMX_NESTED_PAGING(pVM) \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
3461
3462	/* Apply the hardware specified CR0 fixed bits and enable caching. */
3463	u64GuestCr0 \|= fSetCr0;
3464	u64GuestCr0 &= fZapCr0;
3465	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
3466
3467	/* Commit the CR0 and related fields to the guest VMCS. */
3468	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc);
3469	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc);
3470	if (uProcCtls != pVmcsInfo->u32ProcCtls)
3471	{
3472	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3473	AssertRC(rc);
3474	}
3475	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
3476	{
3477	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3478	AssertRC(rc);
3479	}
3480
3481	/* Update our caches. */
3482	pVmcsInfo->u32ProcCtls = uProcCtls;
3483	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3484
3485	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
3486	}
3487	else
3488	{
3489	/*
3490	* With nested-guests, we may have extended the guest/host mask here since we
3491	* merged in the outer guest's mask. Thus, the merged mask can include more bits
3492	* (to read from the nested-guest CR0 read-shadow) than the nested hypervisor
3493	* originally supplied. We must copy those bits from the nested-guest CR0 into
3494	* the nested-guest CR0 read-shadow.
3495	*/
3496	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3497	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
3498	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(&pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
3499	Assert(!RT_HI_U32(u64GuestCr0));
3500	Assert(u64GuestCr0 & X86_CR0_NE);
3501
3502	/* Apply the hardware specified CR0 fixed bits and enable caching. */
3503	u64GuestCr0 \|= fSetCr0;
3504	u64GuestCr0 &= fZapCr0;
3505	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
3506
3507	/* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
3508	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc);
3509	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc);
3510
3511	Log4Func(("cr0=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
3512	}
3513
3514	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3515	}
3516
3517	return VINF_SUCCESS;
3518	}
3519
3520
3521	/**
3522	* Exports the guest control registers (CR3, CR4) into the guest-state area
3523	* in the VMCS.
3524	*
3525	* @returns VBox strict status code.
3526	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3527	* without unrestricted guest access and the VMMDev is not presently
3528	* mapped (e.g. EFI32).
3529	*
3530	* @param pVCpu The cross context virtual CPU structure.
3531	* @param pVmxTransient The VMX-transient structure.
3532	*
3533	* @remarks No-long-jump zone!!!
3534	*/
3535	static VBOXSTRICTRC vmxHCExportGuestCR3AndCR4(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
3536	{
3537	int rc = VINF_SUCCESS;
3538	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3539
3540	/*
3541	* Guest CR2.
3542	* It's always loaded in the assembler code. Nothing to do here.
3543	*/
3544
3545	/*
3546	* Guest CR3.
3547	*/
3548	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CR3)
3549	{
3550	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
3551
3552	if (VM_IS_VMX_NESTED_PAGING(pVM))
3553	{
3554	#ifdef IN_RING0
3555	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
3556	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3557
3558	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3559	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
3560	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3561	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
3562
3563	/* VMX_EPT_MEMTYPE_WB support is already checked in vmxHCSetupTaggedTlb(). */
3564	pVmcsInfo->HCPhysEPTP \|= RT_BF_MAKE(VMX_BF_EPTP_MEMTYPE, VMX_EPTP_MEMTYPE_WB)
3565	\| RT_BF_MAKE(VMX_BF_EPTP_PAGE_WALK_LENGTH, VMX_EPTP_PAGE_WALK_LENGTH_4);
3566
3567	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3568	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3569	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3570	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
3571	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3572	\|\| (g_HmMsrs.u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_ACCESS_DIRTY),
3573	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
3574
3575	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
3576	AssertRC(rc);
3577	#endif
3578
3579	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3580	uint64_t u64GuestCr3 = pCtx->cr3;
3581	if ( VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
3582	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
3583	{
3584	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3585	if (CPUMIsGuestInPAEModeEx(pCtx))
3586	{
3587	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, pCtx->aPaePdpes[0].u); AssertRC(rc);
3588	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, pCtx->aPaePdpes[1].u); AssertRC(rc);
3589	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, pCtx->aPaePdpes[2].u); AssertRC(rc);
3590	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, pCtx->aPaePdpes[3].u); AssertRC(rc);
3591	}
3592
3593	/*
3594	* The guest's view of its CR3 is unblemished with nested paging when the
3595	* guest is using paging or we have unrestricted guest execution to handle
3596	* the guest when it's not using paging.
3597	*/
3598	}
3599	#ifdef IN_RING0
3600	else
3601	{
3602	/*
3603	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
3604	* thinks it accesses physical memory directly, we use our identity-mapped
3605	* page table to map guest-linear to guest-physical addresses. EPT takes care
3606	* of translating it to host-physical addresses.
3607	*/
3608	RTGCPHYS GCPhys;
3609	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3610
3611	/* We obtain it here every time as the guest could have relocated this PCI region. */
3612	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3613	if (RT_SUCCESS(rc))
3614	{ /* likely */ }
3615	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3616	{
3617	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3618	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3619	}
3620	else
3621	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3622
3623	u64GuestCr3 = GCPhys;
3624	}
3625	#endif
3626
3627	Log4Func(("guest_cr3=%#RX64 (GstN)\n", u64GuestCr3));
3628	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR3, u64GuestCr3);
3629	AssertRC(rc);
3630	}
3631	else
3632	{
3633	Assert(!pVmxTransient->fIsNestedGuest);
3634	/* Non-nested paging case, just use the hypervisor's CR3. */
3635	RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);
3636
3637	Log4Func(("guest_cr3=%#RX64 (HstN)\n", HCPhysGuestCr3));
3638	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
3639	AssertRC(rc);
3640	}
3641
3642	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3643	}
3644
3645	/*
3646	* Guest CR4.
3647	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
3648	*/
3649	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CR4)
3650	{
3651	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3652	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
3653
3654	uint64_t const fSetCr4 = g_HmMsrs.u.vmx.u64Cr4Fixed0;
3655	uint64_t const fZapCr4 = g_HmMsrs.u.vmx.u64Cr4Fixed1;
3656
3657	/*
3658	* With nested-guests, we may have extended the guest/host mask here (since we
3659	* merged in the outer guest's mask, see vmxHCMergeVmcsNested). This means, the
3660	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
3661	* the nested hypervisor originally supplied. Thus, we should, in essence, copy
3662	* those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
3663	*/
3664	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
3665	uint64_t u64GuestCr4 = pCtx->cr4;
3666	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
3667	? pCtx->cr4
3668	: CPUMGetGuestVmxMaskedCr4(pCtx, pVmcsInfo->u64Cr4Mask);
3669	Assert(!RT_HI_U32(u64GuestCr4));
3670
3671	#ifdef IN_RING0
3672	/*
3673	* Setup VT-x's view of the guest CR4.
3674	*
3675	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3676	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3677	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3678	*
3679	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3680	*/
3681	if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
3682	{
3683	Assert(pVM->hm.s.vmx.pRealModeTSS);
3684	Assert(PDMVmmDevHeapIsEnabled(pVM));
3685	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
3686	}
3687	#endif
3688
3689	if (VM_IS_VMX_NESTED_PAGING(pVM))
3690	{
3691	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3692	&& !VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
3693	{
3694	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3695	u64GuestCr4 \|= X86_CR4_PSE;
3696	/* Our identity mapping is a 32-bit page directory. */
3697	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
3698	}
3699	/* else use guest CR4.*/
3700	}
3701	else
3702	{
3703	Assert(!pVmxTransient->fIsNestedGuest);
3704
3705	/*
3706	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3707	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3708	*/
3709	switch (VCPU_2_VMXSTATE(pVCpu).enmShadowMode)
3710	{
3711	case PGMMODE_REAL: /* Real-mode. */
3712	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3713	case PGMMODE_32_BIT: /* 32-bit paging. */
3714	{
3715	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
3716	break;
3717	}
3718
3719	case PGMMODE_PAE: /* PAE paging. */
3720	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3721	{
3722	u64GuestCr4 \|= X86_CR4_PAE;
3723	break;
3724	}
3725
3726	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3727	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3728	{
3729	#ifdef VBOX_WITH_64_BITS_GUESTS
3730	/* For our assumption in vmxHCShouldSwapEferMsr. */
3731	Assert(u64GuestCr4 & X86_CR4_PAE);
3732	break;
3733	#endif
3734	}
3735	default:
3736	AssertFailed();
3737	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3738	}
3739	}
3740
3741	/* Apply the hardware specified CR4 fixed bits (mainly CR4.VMXE). */
3742	u64GuestCr4 \|= fSetCr4;
3743	u64GuestCr4 &= fZapCr4;
3744
3745	/* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
3746	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR4, u64GuestCr4); AssertRC(rc);
3747	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4); AssertRC(rc);
3748
3749	#ifdef IN_RING0
3750	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
3751	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
3752	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
3753	{
3754	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
3755	vmxHCUpdateStartVmFunction(pVCpu);
3756	}
3757	#endif
3758
3759	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CR4);
3760
3761	Log4Func(("cr4=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
3762	}
3763	return rc;
3764	}
3765
3766
3767	#ifdef IN_RING0
3768	/**
3769	* Exports the guest debug registers into the guest-state area in the VMCS.
3770	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3771	*
3772	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
3773	*
3774	* @returns VBox status code.
3775	* @param pVCpu The cross context virtual CPU structure.
3776	* @param pVmxTransient The VMX-transient structure.
3777	*
3778	* @remarks No-long-jump zone!!!
3779	*/
3780	static int vmxHCExportSharedDebugState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
3781	{
3782	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3783
3784	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
3785	* stepping. */
3786	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
3787	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3788	if (pVmxTransient->fIsNestedGuest)
3789	{
3790	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
3791	AssertRC(rc);
3792
3793	/*
3794	* We don't want to always intercept MOV DRx for nested-guests as it causes
3795	* problems when the nested hypervisor isn't intercepting them, see @bugref{10080}.
3796	* Instead, they are strictly only requested when the nested hypervisor intercepts
3797	* them -- handled while merging VMCS controls.
3798	*
3799	* If neither the outer nor the nested-hypervisor is intercepting MOV DRx,
3800	* then the nested-guest debug state should be actively loaded on the host so that
3801	* nested-guest reads its own debug registers without causing VM-exits.
3802	*/
3803	if ( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT)
3804	&& !CPUMIsGuestDebugStateActive(pVCpu))
3805	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3806	return VINF_SUCCESS;
3807	}
3808	#endif
3809
3810	#ifdef VBOX_STRICT
3811	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3812	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
3813	{
3814	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3815	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
3816	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
3817	}
3818	#endif
3819
3820	bool fSteppingDB = false;
3821	bool fInterceptMovDRx = false;
3822	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
3823	if (VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
3824	{
3825	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3826	if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
3827	{
3828	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
3829	Assert(fSteppingDB == false);
3830	}
3831	else
3832	{
3833	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
3834	VCPU_2_VMXSTATE(pVCpu).fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
3835	pVCpu->hmr0.s.fClearTrapFlag = true;
3836	fSteppingDB = true;
3837	}
3838	}
3839
3840	uint64_t u64GuestDr7;
3841	if ( fSteppingDB
3842	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3843	{
3844	/*
3845	* Use the combined guest and host DRx values found in the hypervisor register set
3846	* because the hypervisor debugger has breakpoints active or someone is single stepping
3847	* on the host side without a monitor trap flag.
3848	*
3849	* Note! DBGF expects a clean DR6 state before executing guest code.
3850	*/
3851	if (!CPUMIsHyperDebugStateActive(pVCpu))
3852	{
3853	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3854	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3855	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3856	}
3857
3858	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
3859	u64GuestDr7 = CPUMGetHyperDR7(pVCpu);
3860	pVCpu->hmr0.s.fUsingHyperDR7 = true;
3861	fInterceptMovDRx = true;
3862	}
3863	else
3864	{
3865	/*
3866	* If the guest has enabled debug registers, we need to load them prior to
3867	* executing guest code so they'll trigger at the right time.
3868	*/
3869	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
3870	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3871	{
3872	if (!CPUMIsGuestDebugStateActive(pVCpu))
3873	{
3874	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3875	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3876	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3877	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatDRxArmed);
3878	}
3879	Assert(!fInterceptMovDRx);
3880	}
3881	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3882	{
3883	/*
3884	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3885	* must intercept #DB in order to maintain a correct DR6 guest value, and
3886	* because we need to intercept it to prevent nested #DBs from hanging the
3887	* CPU, we end up always having to intercept it. See vmxHCSetupVmcsXcptBitmap().
3888	*/
3889	fInterceptMovDRx = true;
3890	}
3891
3892	/* Update DR7 with the actual guest value. */
3893	u64GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
3894	pVCpu->hmr0.s.fUsingHyperDR7 = false;
3895	}
3896
3897	if (fInterceptMovDRx)
3898	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
3899	else
3900	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
3901
3902	/*
3903	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
3904	* monitor-trap flag and update our cache.
3905	*/
3906	if (uProcCtls != pVmcsInfo->u32ProcCtls)
3907	{
3908	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3909	AssertRC(rc);
3910	pVmcsInfo->u32ProcCtls = uProcCtls;
3911	}
3912
3913	/*
3914	* Update guest DR7.
3915	*/
3916	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_DR7, u64GuestDr7);
3917	AssertRC(rc);
3918
3919	/*
3920	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
3921	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
3922	*
3923	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
3924	*/
3925	if (fSteppingDB)
3926	{
3927	Assert(VCPU_2_VMXSTATE(pVCpu).fSingleInstruction);
3928	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
3929
3930	uint32_t fIntrState = 0;
3931	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
3932	AssertRC(rc);
3933
3934	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
3935	{
3936	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
3937	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, fIntrState);
3938	AssertRC(rc);
3939	}
3940	}
3941
3942	return VINF_SUCCESS;
3943	}
3944	#endif /* !IN_RING0 */
3945
3946
3947	#ifdef VBOX_STRICT
3948	/**
3949	* Strict function to validate segment registers.
3950	*
3951	* @param pVCpu The cross context virtual CPU structure.
3952	* @param pVmcsInfo The VMCS info. object.
3953	*
3954	* @remarks Will import guest CR0 on strict builds during validation of
3955	* segments.
3956	*/
3957	static void vmxHCValidateSegmentRegs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3958	{
3959	/*
3960	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
3961	*
3962	* The reason we check for attribute value 0 in this function and not just the unusable bit is
3963	* because vmxHCExportGuestSegReg() only updates the VMCS' copy of the value with the
3964	* unusable bit and doesn't change the guest-context value.
3965	*/
3966	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3967	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3968	vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
3969	if ( !VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
3970	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3971	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3972	{
3973	/* Protected mode checks */
3974	/* CS */
3975	Assert(pCtx->cs.Attr.n.u1Present);
3976	Assert(!(pCtx->cs.Attr.u & 0xf00));
3977	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3978	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3979	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3980	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3981	\|\| (pCtx->cs.Attr.n.u1Granularity));
3982	/* CS cannot be loaded with NULL in protected mode. */
3983	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
3984	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3985	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3986	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3987	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3988	else
3989	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3990	/* SS */
3991	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3992	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3993	if ( !(pCtx->cr0 & X86_CR0_PE)
3994	\|\| pCtx->cs.Attr.n.u4Type == 3)
3995	{
3996	Assert(!pCtx->ss.Attr.n.u2Dpl);
3997	}
3998	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3999	{
4000	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4001	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4002	Assert(pCtx->ss.Attr.n.u1Present);
4003	Assert(!(pCtx->ss.Attr.u & 0xf00));
4004	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4005	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4006	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4007	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4008	\|\| (pCtx->ss.Attr.n.u1Granularity));
4009	}
4010	/* DS, ES, FS, GS - only check for usable selectors, see vmxHCExportGuestSegReg(). */
4011	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4012	{
4013	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4014	Assert(pCtx->ds.Attr.n.u1Present);
4015	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4016	Assert(!(pCtx->ds.Attr.u & 0xf00));
4017	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4018	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4019	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4020	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4021	\|\| (pCtx->ds.Attr.n.u1Granularity));
4022	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4023	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4024	}
4025	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4026	{
4027	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4028	Assert(pCtx->es.Attr.n.u1Present);
4029	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4030	Assert(!(pCtx->es.Attr.u & 0xf00));
4031	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4032	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4033	\|\| !(pCtx->es.Attr.n.u1Granularity));
4034	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4035	\|\| (pCtx->es.Attr.n.u1Granularity));
4036	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4037	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4038	}
4039	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4040	{
4041	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4042	Assert(pCtx->fs.Attr.n.u1Present);
4043	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4044	Assert(!(pCtx->fs.Attr.u & 0xf00));
4045	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4046	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4047	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4048	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4049	\|\| (pCtx->fs.Attr.n.u1Granularity));
4050	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4051	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4052	}
4053	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4054	{
4055	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4056	Assert(pCtx->gs.Attr.n.u1Present);
4057	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4058	Assert(!(pCtx->gs.Attr.u & 0xf00));
4059	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4060	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4061	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4062	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4063	\|\| (pCtx->gs.Attr.n.u1Granularity));
4064	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4065	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4066	}
4067	/* 64-bit capable CPUs. */
4068	Assert(!RT_HI_U32(pCtx->cs.u64Base));
4069	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
4070	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
4071	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
4072	}
4073	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4074	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4075	&& !VM_IS_VMX_UNRESTRICTED_GUEST(pVM)))
4076	{
4077	/* Real and v86 mode checks. */
4078	/* vmxHCExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4079	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4080	#ifdef IN_RING0
4081	if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
4082	{
4083	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
4084	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4085	}
4086	else
4087	#endif
4088	{
4089	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4090	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4091	}
4092
4093	/* CS */
4094	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4095	Assert(pCtx->cs.u32Limit == 0xffff);
4096	Assert(u32CSAttr == 0xf3);
4097	/* SS */
4098	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4099	Assert(pCtx->ss.u32Limit == 0xffff);
4100	Assert(u32SSAttr == 0xf3);
4101	/* DS */
4102	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4103	Assert(pCtx->ds.u32Limit == 0xffff);
4104	Assert(u32DSAttr == 0xf3);
4105	/* ES */
4106	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4107	Assert(pCtx->es.u32Limit == 0xffff);
4108	Assert(u32ESAttr == 0xf3);
4109	/* FS */
4110	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4111	Assert(pCtx->fs.u32Limit == 0xffff);
4112	Assert(u32FSAttr == 0xf3);
4113	/* GS */
4114	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4115	Assert(pCtx->gs.u32Limit == 0xffff);
4116	Assert(u32GSAttr == 0xf3);
4117	/* 64-bit capable CPUs. */
4118	Assert(!RT_HI_U32(pCtx->cs.u64Base));
4119	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
4120	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
4121	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
4122	}
4123	}
4124	#endif /* VBOX_STRICT */
4125
4126
4127	/**
4128	* Exports a guest segment register into the guest-state area in the VMCS.
4129	*
4130	* @returns VBox status code.
4131	* @param pVCpu The cross context virtual CPU structure.
4132	* @param pVmcsInfo The VMCS info. object.
4133	* @param iSegReg The segment register number (X86_SREG_XXX).
4134	* @param pSelReg Pointer to the segment selector.
4135	*
4136	* @remarks No-long-jump zone!!!
4137	*/
4138	static int vmxHCExportGuestSegReg(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, uint32_t iSegReg, PCCPUMSELREG pSelReg)
4139	{
4140	Assert(iSegReg < X86_SREG_COUNT);
4141
4142	uint32_t u32Access = pSelReg->Attr.u;
4143	#ifdef IN_RING0
4144	if (!pVmcsInfo->pShared->RealMode.fRealOnV86Active)
4145	#endif
4146	{
4147	/*
4148	* The way to differentiate between whether this is really a null selector or was just
4149	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
4150	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4151	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4152	* NULL selectors loaded in protected-mode have their attribute as 0.
4153	*/
4154	if (u32Access)
4155	{ }
4156	else
4157	u32Access = X86DESCATTR_UNUSABLE;
4158	}
4159	#ifdef IN_RING0
4160	else
4161	{
4162	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4163	u32Access = 0xf3;
4164	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4165	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4166	RT_NOREF_PV(pVCpu);
4167	}
4168	#else
4169	RT_NOREF(pVmcsInfo);
4170	#endif
4171
4172	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4173	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4174	("Access bit not set for usable segment. %.2s sel=%#x attr %#x\n", "ESCSSSDSFSGS" + iSegReg * 2, pSelReg, pSelReg->Attr.u));
4175
4176	/*
4177	* Commit it to the VMCS.
4178	*/
4179	Assert((uint32_t)VMX_VMCS16_GUEST_SEG_SEL(iSegReg) == g_aVmcsSegSel[iSegReg]);
4180	Assert((uint32_t)VMX_VMCS32_GUEST_SEG_LIMIT(iSegReg) == g_aVmcsSegLimit[iSegReg]);
4181	Assert((uint32_t)VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(iSegReg) == g_aVmcsSegAttr[iSegReg]);
4182	Assert((uint32_t)VMX_VMCS_GUEST_SEG_BASE(iSegReg) == g_aVmcsSegBase[iSegReg]);
4183	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS16_GUEST_SEG_SEL(iSegReg), pSelReg->Sel); AssertRC(rc);
4184	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_SEG_LIMIT(iSegReg), pSelReg->u32Limit); AssertRC(rc);
4185	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_SEG_BASE(iSegReg), pSelReg->u64Base); AssertRC(rc);
4186	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(iSegReg), u32Access); AssertRC(rc);
4187	return VINF_SUCCESS;
4188	}
4189
4190
4191	/**
4192	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
4193	* area in the VMCS.
4194	*
4195	* @returns VBox status code.
4196	* @param pVCpu The cross context virtual CPU structure.
4197	* @param pVmxTransient The VMX-transient structure.
4198	*
4199	* @remarks Will import guest CR0 on strict builds during validation of
4200	* segments.
4201	* @remarks No-long-jump zone!!!
4202	*/
4203	static int vmxHCExportGuestSegRegsXdtr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
4204	{
4205	int rc = VERR_INTERNAL_ERROR_5;
4206	#ifdef IN_RING0
4207	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4208	#endif
4209	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4210	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4211	#ifdef IN_RING0
4212	PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
4213	#endif
4214
4215	/*
4216	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4217	*/
4218	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4219	{
4220	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CS)
4221	{
4222	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
4223	#ifdef IN_RING0
4224	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
4225	pVmcsInfoShared->RealMode.AttrCS.u = pCtx->cs.Attr.u;
4226	#endif
4227	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
4228	AssertRC(rc);
4229	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CS);
4230	}
4231
4232	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_SS)
4233	{
4234	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
4235	#ifdef IN_RING0
4236	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
4237	pVmcsInfoShared->RealMode.AttrSS.u = pCtx->ss.Attr.u;
4238	#endif
4239	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
4240	AssertRC(rc);
4241	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_SS);
4242	}
4243
4244	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_DS)
4245	{
4246	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
4247	#ifdef IN_RING0
4248	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
4249	pVmcsInfoShared->RealMode.AttrDS.u = pCtx->ds.Attr.u;
4250	#endif
4251	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
4252	AssertRC(rc);
4253	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_DS);
4254	}
4255
4256	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_ES)
4257	{
4258	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
4259	#ifdef IN_RING0
4260	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
4261	pVmcsInfoShared->RealMode.AttrES.u = pCtx->es.Attr.u;
4262	#endif
4263	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
4264	AssertRC(rc);
4265	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_ES);
4266	}
4267
4268	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_FS)
4269	{
4270	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
4271	#ifdef IN_RING0
4272	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
4273	pVmcsInfoShared->RealMode.AttrFS.u = pCtx->fs.Attr.u;
4274	#endif
4275	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
4276	AssertRC(rc);
4277	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_FS);
4278	}
4279
4280	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_GS)
4281	{
4282	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
4283	#ifdef IN_RING0
4284	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
4285	pVmcsInfoShared->RealMode.AttrGS.u = pCtx->gs.Attr.u;
4286	#endif
4287	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
4288	AssertRC(rc);
4289	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_GS);
4290	}
4291
4292	#ifdef VBOX_STRICT
4293	vmxHCValidateSegmentRegs(pVCpu, pVmcsInfo);
4294	#endif
4295	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
4296	pCtx->cs.Attr.u));
4297	}
4298
4299	/*
4300	* Guest TR.
4301	*/
4302	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_TR)
4303	{
4304	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
4305
4306	/*
4307	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4308	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4309	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4310	*/
4311	uint16_t u16Sel;
4312	uint32_t u32Limit;
4313	uint64_t u64Base;
4314	uint32_t u32AccessRights;
4315	#ifdef IN_RING0
4316	if (!pVmcsInfoShared->RealMode.fRealOnV86Active)
4317	#endif
4318	{
4319	u16Sel = pCtx->tr.Sel;
4320	u32Limit = pCtx->tr.u32Limit;
4321	u64Base = pCtx->tr.u64Base;
4322	u32AccessRights = pCtx->tr.Attr.u;
4323	}
4324	#ifdef IN_RING0
4325	else
4326	{
4327	Assert(!pVmxTransient->fIsNestedGuest);
4328	Assert(pVM->hm.s.vmx.pRealModeTSS);
4329	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
4330
4331	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4332	RTGCPHYS GCPhys;
4333	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4334	AssertRCReturn(rc, rc);
4335
4336	X86DESCATTR DescAttr;
4337	DescAttr.u = 0;
4338	DescAttr.n.u1Present = 1;
4339	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4340
4341	u16Sel = 0;
4342	u32Limit = HM_VTX_TSS_SIZE;
4343	u64Base = GCPhys;
4344	u32AccessRights = DescAttr.u;
4345	}
4346	#endif
4347
4348	/* Validate. */
4349	Assert(!(u16Sel & RT_BIT(2)));
4350	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4351	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4352	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4353	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4354	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4355	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4356	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4357	Assert( (u32Limit & 0xfff) == 0xfff
4358	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4359	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
4360	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4361
4362	rc = VMX_VMCS_WRITE_16(pVCpu, VMX_VMCS16_GUEST_TR_SEL, u16Sel); AssertRC(rc);
4363	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRC(rc);
4364	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRC(rc);
4365	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRC(rc);
4366
4367	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_TR);
4368	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
4369	}
4370
4371	/*
4372	* Guest GDTR.
4373	*/
4374	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4375	{
4376	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
4377
4378	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt); AssertRC(rc);
4379	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt); AssertRC(rc);
4380
4381	/* Validate. */
4382	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4383
4384	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4385	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
4386	}
4387
4388	/*
4389	* Guest LDTR.
4390	*/
4391	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4392	{
4393	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
4394
4395	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4396	uint32_t u32Access;
4397	if ( !pVmxTransient->fIsNestedGuest
4398	&& !pCtx->ldtr.Attr.u)
4399	u32Access = X86DESCATTR_UNUSABLE;
4400	else
4401	u32Access = pCtx->ldtr.Attr.u;
4402
4403	rc = VMX_VMCS_WRITE_16(pVCpu, VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel); AssertRC(rc);
4404	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit); AssertRC(rc);
4405	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRC(rc);
4406	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base); AssertRC(rc);
4407
4408	/* Validate. */
4409	if (!(u32Access & X86DESCATTR_UNUSABLE))
4410	{
4411	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4412	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4413	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4414	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4415	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4416	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4417	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
4418	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4419	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
4420	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4421	}
4422
4423	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4424	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
4425	}
4426
4427	/*
4428	* Guest IDTR.
4429	*/
4430	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4431	{
4432	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
4433
4434	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt); AssertRC(rc);
4435	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt); AssertRC(rc);
4436
4437	/* Validate. */
4438	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4439
4440	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4441	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
4442	}
4443
4444	return VINF_SUCCESS;
4445	}
4446
4447
4448	#ifdef IN_RING0
4449	/**
4450	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4451	* areas.
4452	*
4453	* These MSRs will automatically be loaded to the host CPU on every successful
4454	* VM-entry and stored from the host CPU on every successful VM-exit.
4455	*
4456	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
4457	* actual host MSR values are not- updated here for performance reasons. See
4458	* vmxHCExportHostMsrs().
4459	*
4460	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4461	*
4462	* @returns VBox status code.
4463	* @param pVCpu The cross context virtual CPU structure.
4464	* @param pVmxTransient The VMX-transient structure.
4465	*
4466	* @remarks No-long-jump zone!!!
4467	*/
4468	static int vmxHCExportGuestMsrs(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
4469	{
4470	AssertPtr(pVCpu);
4471	AssertPtr(pVmxTransient);
4472
4473	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4474	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4475
4476	/*
4477	* MSRs that we use the auto-load/store MSR area in the VMCS.
4478	* For 64-bit hosts, we load/restore them lazily, see vmxHCLazyLoadGuestMsrs(),
4479	* nothing to do here. The host MSR values are updated when it's safe in
4480	* vmxHCLazySaveHostMsrs().
4481	*
4482	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
4483	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
4484	* emulation. The merged MSR permission bitmap will ensure that we get VM-exits
4485	* for any MSR that are not part of the lazy MSRs so we do not need to place
4486	* those MSRs into the auto-load/store MSR area. Nothing to do here.
4487	*/
4488	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4489	{
4490	/* No auto-load/store MSRs currently. */
4491	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4492	}
4493
4494	/*
4495	* Guest Sysenter MSRs.
4496	*/
4497	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4498	{
4499	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4500
4501	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4502	{
4503	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
4504	AssertRC(rc);
4505	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4506	}
4507
4508	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4509	{
4510	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
4511	AssertRC(rc);
4512	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4513	}
4514
4515	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4516	{
4517	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
4518	AssertRC(rc);
4519	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4520	}
4521	}
4522
4523	/*
4524	* Guest/host EFER MSR.
4525	*/
4526	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4527	{
4528	/* Whether we are using the VMCS to swap the EFER MSR must have been
4529	determined earlier while exporting VM-entry/VM-exit controls. */
4530	Assert(!(ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
4531	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
4532
4533	if (vmxHCShouldSwapEferMsr(pVCpu, pVmxTransient))
4534	{
4535	/*
4536	* EFER.LME is written by software, while EFER.LMA is set by the CPU to (CR0.PG & EFER.LME).
4537	* This means a guest can set EFER.LME=1 while CR0.PG=0 and EFER.LMA can remain 0.
4538	* VT-x requires that "IA-32e mode guest" VM-entry control must be identical to EFER.LMA
4539	* and to CR0.PG. Without unrestricted execution, CR0.PG (used for VT-x, not the shadow)
4540	* must always be 1. This forces us to effectively clear both EFER.LMA and EFER.LME until
4541	* the guest has also set CR0.PG=1. Otherwise, we would run into an invalid-guest state
4542	* during VM-entry.
4543	*/
4544	uint64_t uGuestEferMsr = pCtx->msrEFER;
4545	if (!VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
4546	{
4547	if (!(pCtx->msrEFER & MSR_K6_EFER_LMA))
4548	uGuestEferMsr &= ~MSR_K6_EFER_LME;
4549	else
4550	Assert((pCtx->msrEFER & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
4551	}
4552
4553	/*
4554	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4555	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4556	*/
4557	if (g_fHmVmxSupportsVmcsEfer)
4558	{
4559	int rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_EFER_FULL, uGuestEferMsr);
4560	AssertRC(rc);
4561	}
4562	else
4563	{
4564	/*
4565	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
4566	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
4567	*/
4568	int rc = vmxHCAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, uGuestEferMsr,
4569	false /* fSetReadWrite /, false / fUpdateHostMsr */);
4570	AssertRCReturn(rc, rc);
4571	}
4572
4573	Log4Func(("efer=%#RX64 shadow=%#RX64\n", uGuestEferMsr, pCtx->msrEFER));
4574	}
4575	else if (!g_fHmVmxSupportsVmcsEfer)
4576	vmxHCRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
4577
4578	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4579	}
4580
4581	/*
4582	* Other MSRs.
4583	*/
4584	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
4585	{
4586	/* Speculation Control (R/W). */
4587	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
4588	if (pVM->cpum.ro.GuestFeatures.fIbrs)
4589	{
4590	int rc = vmxHCAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
4591	false /* fSetReadWrite /, false / fUpdateHostMsr */);
4592	AssertRCReturn(rc, rc);
4593	}
4594
4595	/* Last Branch Record. */
4596	if (VM_IS_VMX_LBR(pVM))
4597	{
4598	PVMXVMCSINFOSHARED const pVmcsInfoShared = pVmxTransient->pVmcsInfo->pShared;
4599	uint32_t const idFromIpMsrStart = pVM->hmr0.s.vmx.idLbrFromIpMsrFirst;
4600	uint32_t const idToIpMsrStart = pVM->hmr0.s.vmx.idLbrToIpMsrFirst;
4601	uint32_t const cLbrStack = pVM->hmr0.s.vmx.idLbrFromIpMsrLast - pVM->hmr0.s.vmx.idLbrFromIpMsrFirst + 1;
4602	Assert(cLbrStack <= 32);
4603	for (uint32_t i = 0; i < cLbrStack; i++)
4604	{
4605	int rc = vmxHCAddAutoLoadStoreMsr(pVCpu, pVmxTransient, idFromIpMsrStart + i,
4606	pVmcsInfoShared->au64LbrFromIpMsr[i],
4607	false /* fSetReadWrite /, false / fUpdateHostMsr */);
4608	AssertRCReturn(rc, rc);
4609
4610	/* Some CPUs don't have a Branch-To-IP MSR (P4 and related Xeons). */
4611	if (idToIpMsrStart != 0)
4612	{
4613	rc = vmxHCAddAutoLoadStoreMsr(pVCpu, pVmxTransient, idToIpMsrStart + i,
4614	pVmcsInfoShared->au64LbrToIpMsr[i],
4615	false /* fSetReadWrite /, false / fUpdateHostMsr */);
4616	AssertRCReturn(rc, rc);
4617	}
4618	}
4619
4620	/* Add LBR top-of-stack MSR (which contains the index to the most recent record). */
4621	int rc = vmxHCAddAutoLoadStoreMsr(pVCpu, pVmxTransient, pVM->hmr0.s.vmx.idLbrTosMsr,
4622	pVmcsInfoShared->u64LbrTosMsr, false /* fSetReadWrite */,
4623	false /* fUpdateHostMsr */);
4624	AssertRCReturn(rc, rc);
4625	}
4626
4627	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
4628	}
4629
4630	return VINF_SUCCESS;
4631	}
4632
4633
4634	/**
4635	* Sets up the usage of TSC-offsetting and updates the VMCS.
4636	*
4637	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
4638	* VMX-preemption timer.
4639	*
4640	* @returns VBox status code.
4641	* @param pVCpu The cross context virtual CPU structure.
4642	* @param pVmxTransient The VMX-transient structure.
4643	* @param idCurrentCpu The current CPU number.
4644	*
4645	* @remarks No-long-jump zone!!!
4646	*/
4647	static void vmxHCUpdateTscOffsettingAndPreemptTimer(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, RTCPUID idCurrentCpu)
4648	{
4649	bool fOffsettedTsc;
4650	bool fParavirtTsc;
4651	uint64_t uTscOffset;
4652	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4653	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
4654
4655	if (VM_IS_VMX_PREEMPT_TIMER_USED(pVM))
4656	{
4657	/* The TMCpuTickGetDeadlineAndTscOffset function is expensive (calling it on
4658	every entry slowed down the bs2-test1 CPUID testcase by ~33% (on an 10980xe). */
4659	uint64_t cTicksToDeadline;
4660	if ( idCurrentCpu == pVCpu->hmr0.s.idLastCpu
4661	&& TMVirtualSyncIsCurrentDeadlineVersion(pVM, pVCpu->hmr0.s.vmx.uTscDeadlineVersion))
4662	{
4663	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatVmxPreemptionReusingDeadline);
4664	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
4665	cTicksToDeadline = pVCpu->hmr0.s.vmx.uTscDeadline - SUPReadTsc();
4666	if ((int64_t)cTicksToDeadline > 0)
4667	{ /* hopefully */ }
4668	else
4669	{
4670	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatVmxPreemptionReusingDeadlineExpired);
4671	cTicksToDeadline = 0;
4672	}
4673	}
4674	else
4675	{
4676	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatVmxPreemptionRecalcingDeadline);
4677	cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc,
4678	&pVCpu->hmr0.s.vmx.uTscDeadline,
4679	&pVCpu->hmr0.s.vmx.uTscDeadlineVersion);
4680	pVCpu->hmr0.s.vmx.uTscDeadline += cTicksToDeadline;
4681	if (cTicksToDeadline >= 128)
4682	{ /* hopefully */ }
4683	else
4684	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatVmxPreemptionRecalcingDeadlineExpired);
4685	}
4686
4687	/* Make sure the returned values have sane upper and lower boundaries. */
4688	uint64_t const u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
4689	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second, 15.625ms. / /* @todo r=bird: Once real+virtual timers move to separate thread, we can raise the upper limit (16ms isn't much). ASSUMES working poke cpu function. */
4690	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 32678); /* 1/32768th of a second, ~30us. */
4691	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4692
4693	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
4694	* preemption timers here. We probably need to clamp the preemption timer,
4695	* after converting the timer value to the host. */
4696	uint32_t const cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4697	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
4698	AssertRC(rc);
4699	}
4700	else
4701	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
4702
4703	if (fParavirtTsc)
4704	{
4705	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
4706	information before every VM-entry, hence disable it for performance sake. */
4707	#if 0
4708	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
4709	AssertRC(rc);
4710	#endif
4711	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatTscParavirt);
4712	}
4713
4714	if ( fOffsettedTsc
4715	&& RT_LIKELY(!pVCpu->hmr0.s.fDebugWantRdTscExit))
4716	{
4717	if (pVmxTransient->fIsNestedGuest)
4718	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
4719	vmxHCSetTscOffsetVmcs(pVCpu, pVmcsInfo, uTscOffset);
4720	vmxHCRemoveProcCtlsVmcs(pVCpu, pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
4721	}
4722	else
4723	{
4724	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4725	vmxHCSetProcCtlsVmcs(pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
4726	}
4727	}
4728	#endif /* !IN_RING0 */
4729
4730
4731	/**
4732	* Gets the IEM exception flags for the specified vector and IDT vectoring /
4733	* VM-exit interruption info type.
4734	*
4735	* @returns The IEM exception flags.
4736	* @param uVector The event vector.
4737	* @param uVmxEventType The VMX event type.
4738	*
4739	* @remarks This function currently only constructs flags required for
4740	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
4741	* and CR2 aspects of an exception are not included).
4742	*/
4743	static uint32_t vmxHCGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
4744	{
4745	uint32_t fIemXcptFlags;
4746	switch (uVmxEventType)
4747	{
4748	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
4749	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
4750	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
4751	break;
4752
4753	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
4754	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
4755	break;
4756
4757	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
4758	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
4759	break;
4760
4761	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
4762	{
4763	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
4764	if (uVector == X86_XCPT_BP)
4765	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
4766	else if (uVector == X86_XCPT_OF)
4767	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
4768	else
4769	{
4770	fIemXcptFlags = 0;
4771	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
4772	}
4773	break;
4774	}
4775
4776	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
4777	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
4778	break;
4779
4780	default:
4781	fIemXcptFlags = 0;
4782	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
4783	break;
4784	}
4785	return fIemXcptFlags;
4786	}
4787
4788
4789	/**
4790	* Sets an event as a pending event to be injected into the guest.
4791	*
4792	* @param pVCpu The cross context virtual CPU structure.
4793	* @param u32IntInfo The VM-entry interruption-information field.
4794	* @param cbInstr The VM-entry instruction length in bytes (for
4795	* software interrupts, exceptions and privileged
4796	* software exceptions).
4797	* @param u32ErrCode The VM-entry exception error code.
4798	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4799	* page-fault.
4800	*/
4801	DECLINLINE(void) vmxHCSetPendingEvent(PVMCPUCC pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4802	RTGCUINTPTR GCPtrFaultAddress)
4803	{
4804	Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending);
4805	VCPU_2_VMXSTATE(pVCpu).Event.fPending = true;
4806	VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo = u32IntInfo;
4807	VCPU_2_VMXSTATE(pVCpu).Event.u32ErrCode = u32ErrCode;
4808	VCPU_2_VMXSTATE(pVCpu).Event.cbInstr = cbInstr;
4809	VCPU_2_VMXSTATE(pVCpu).Event.GCPtrFaultAddress = GCPtrFaultAddress;
4810	}
4811
4812
4813	/**
4814	* Sets an external interrupt as pending-for-injection into the VM.
4815	*
4816	* @param pVCpu The cross context virtual CPU structure.
4817	* @param u8Interrupt The external interrupt vector.
4818	*/
4819	DECLINLINE(void) vmxHCSetPendingExtInt(PVMCPUCC pVCpu, uint8_t u8Interrupt)
4820	{
4821	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
4822	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
4823	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
4824	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
4825	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4826	}
4827
4828
4829	/**
4830	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
4831	*
4832	* @param pVCpu The cross context virtual CPU structure.
4833	*/
4834	DECLINLINE(void) vmxHCSetPendingXcptNmi(PVMCPUCC pVCpu)
4835	{
4836	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
4837	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
4838	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
4839	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
4840	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4841	}
4842
4843
4844	/**
4845	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
4846	*
4847	* @param pVCpu The cross context virtual CPU structure.
4848	*/
4849	DECLINLINE(void) vmxHCSetPendingXcptDF(PVMCPUCC pVCpu)
4850	{
4851	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
4852	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
4853	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
4854	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
4855	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4856	}
4857
4858
4859	/**
4860	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
4861	*
4862	* @param pVCpu The cross context virtual CPU structure.
4863	*/
4864	DECLINLINE(void) vmxHCSetPendingXcptUD(PVMCPUCC pVCpu)
4865	{
4866	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
4867	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
4868	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
4869	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
4870	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4871	}
4872
4873
4874	/**
4875	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
4876	*
4877	* @param pVCpu The cross context virtual CPU structure.
4878	*/
4879	DECLINLINE(void) vmxHCSetPendingXcptDB(PVMCPUCC pVCpu)
4880	{
4881	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
4882	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
4883	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
4884	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
4885	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4886	}
4887
4888
4889	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4890	/**
4891	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
4892	*
4893	* @param pVCpu The cross context virtual CPU structure.
4894	* @param u32ErrCode The error code for the general-protection exception.
4895	*/
4896	DECLINLINE(void) vmxHCSetPendingXcptGP(PVMCPUCC pVCpu, uint32_t u32ErrCode)
4897	{
4898	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
4899	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
4900	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
4901	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
4902	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
4903	}
4904
4905
4906	/**
4907	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
4908	*
4909	* @param pVCpu The cross context virtual CPU structure.
4910	* @param u32ErrCode The error code for the stack exception.
4911	*/
4912	DECLINLINE(void) vmxHCSetPendingXcptSS(PVMCPUCC pVCpu, uint32_t u32ErrCode)
4913	{
4914	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
4915	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
4916	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
4917	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
4918	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
4919	}
4920	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
4921
4922
4923	/**
4924	* Fixes up attributes for the specified segment register.
4925	*
4926	* @param pVCpu The cross context virtual CPU structure.
4927	* @param pSelReg The segment register that needs fixing.
4928	* @param pszRegName The register name (for logging and assertions).
4929	*/
4930	static void vmxHCFixUnusableSegRegAttr(PVMCPUCC pVCpu, PCPUMSELREG pSelReg, const char *pszRegName)
4931	{
4932	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
4933
4934	/*
4935	* If VT-x marks the segment as unusable, most other bits remain undefined:
4936	* - For CS the L, D and G bits have meaning.
4937	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
4938	* - For the remaining data segments no bits are defined.
4939	*
4940	* The present bit and the unusable bit has been observed to be set at the
4941	* same time (the selector was supposed to be invalid as we started executing
4942	* a V8086 interrupt in ring-0).
4943	*
4944	* What should be important for the rest of the VBox code, is that the P bit is
4945	* cleared. Some of the other VBox code recognizes the unusable bit, but
4946	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
4947	* safe side here, we'll strip off P and other bits we don't care about. If
4948	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
4949	*
4950	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
4951	*/
4952	#ifdef VBOX_STRICT
4953	uint32_t const uAttr = pSelReg->Attr.u;
4954	#endif
4955
4956	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
4957	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
4958	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
4959
4960	#ifdef VBOX_STRICT
4961	# ifdef IN_RING0
4962	VMMRZCallRing3Disable(pVCpu);
4963	# endif
4964	Log4Func(("Unusable %s: sel=%#x attr=%#x -> %#x\n", pszRegName, pSelReg->Sel, uAttr, pSelReg->Attr.u));
4965	# ifdef DEBUG_bird
4966	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
4967	("%s: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
4968	pszRegName, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
4969	# endif
4970	# ifdef IN_RING0
4971	VMMRZCallRing3Enable(pVCpu);
4972	# endif
4973	NOREF(uAttr);
4974	#endif
4975	RT_NOREF2(pVCpu, pszRegName);
4976	}
4977
4978
4979	/**
4980	* Imports a guest segment register from the current VMCS into the guest-CPU
4981	* context.
4982	*
4983	* @param pVCpu The cross context virtual CPU structure.
4984	* @param iSegReg The segment register number (X86_SREG_XXX).
4985	*
4986	* @remarks Called with interrupts and/or preemption disabled.
4987	*/
4988	static void vmxHCImportGuestSegReg(PVMCPUCC pVCpu, uint32_t iSegReg)
4989	{
4990	Assert(iSegReg < X86_SREG_COUNT);
4991	Assert((uint32_t)VMX_VMCS16_GUEST_SEG_SEL(iSegReg) == g_aVmcsSegSel[iSegReg]);
4992	Assert((uint32_t)VMX_VMCS32_GUEST_SEG_LIMIT(iSegReg) == g_aVmcsSegLimit[iSegReg]);
4993	Assert((uint32_t)VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(iSegReg) == g_aVmcsSegAttr[iSegReg]);
4994	Assert((uint32_t)VMX_VMCS_GUEST_SEG_BASE(iSegReg) == g_aVmcsSegBase[iSegReg]);
4995
4996	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
4997
4998	uint16_t u16Sel;
4999	int rc = VMX_VMCS_READ_16(pVCpu, VMX_VMCS16_GUEST_SEG_SEL(iSegReg), &u16Sel); AssertRC(rc);
5000	pSelReg->Sel = u16Sel;
5001	pSelReg->ValidSel = u16Sel;
5002
5003	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SEG_LIMIT(iSegReg), &pSelReg->u32Limit); AssertRC(rc);
5004	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SEG_BASE(iSegReg), &pSelReg->u64Base); AssertRC(rc);
5005
5006	uint32_t u32Attr;
5007	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(iSegReg), &u32Attr); AssertRC(rc);
5008	pSelReg->Attr.u = u32Attr;
5009	if (u32Attr & X86DESCATTR_UNUSABLE)
5010	vmxHCFixUnusableSegRegAttr(pVCpu, pSelReg, "ES\0CS\0SS\0DS\0FS\0GS" + iSegReg * 3);
5011
5012	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5013	}
5014
5015
5016	/**
5017	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
5018	*
5019	* @param pVCpu The cross context virtual CPU structure.
5020	*
5021	* @remarks Called with interrupts and/or preemption disabled.
5022	*/
5023	static void vmxHCImportGuestLdtr(PVMCPUCC pVCpu)
5024	{
5025	uint16_t u16Sel;
5026	uint64_t u64Base;
5027	uint32_t u32Limit, u32Attr;
5028	int rc = VMX_VMCS_READ_16(pVCpu, VMX_VMCS16_GUEST_LDTR_SEL, &u16Sel); AssertRC(rc);
5029	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit); AssertRC(rc);
5030	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc);
5031	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_LDTR_BASE, &u64Base); AssertRC(rc);
5032
5033	pVCpu->cpum.GstCtx.ldtr.Sel = u16Sel;
5034	pVCpu->cpum.GstCtx.ldtr.ValidSel = u16Sel;
5035	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
5036	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
5037	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
5038	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
5039	if (u32Attr & X86DESCATTR_UNUSABLE)
5040	vmxHCFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, "LDTR");
5041	}
5042
5043
5044	/**
5045	* Imports the guest TR from the current VMCS into the guest-CPU context.
5046	*
5047	* @param pVCpu The cross context virtual CPU structure.
5048	*
5049	* @remarks Called with interrupts and/or preemption disabled.
5050	*/
5051	static void vmxHCImportGuestTr(PVMCPUCC pVCpu)
5052	{
5053	uint16_t u16Sel;
5054	uint64_t u64Base;
5055	uint32_t u32Limit, u32Attr;
5056	int rc = VMX_VMCS_READ_16(pVCpu, VMX_VMCS16_GUEST_TR_SEL, &u16Sel); AssertRC(rc);
5057	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit); AssertRC(rc);
5058	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc);
5059	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_TR_BASE, &u64Base); AssertRC(rc);
5060
5061	pVCpu->cpum.GstCtx.tr.Sel = u16Sel;
5062	pVCpu->cpum.GstCtx.tr.ValidSel = u16Sel;
5063	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
5064	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
5065	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
5066	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
5067	/* TR is the only selector that can never be unusable. */
5068	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
5069	}
5070
5071
5072	/**
5073	* Imports the guest RIP from the VMCS back into the guest-CPU context.
5074	*
5075	* @param pVCpu The cross context virtual CPU structure.
5076	*
5077	* @remarks Called with interrupts and/or preemption disabled, should not assert!
5078	* @remarks Do -not- call this function directly, use vmxHCImportGuestState()
5079	* instead!!!
5080	*/
5081	static void vmxHCImportGuestRip(PVMCPUCC pVCpu)
5082	{
5083	uint64_t u64Val;
5084	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5085	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
5086	{
5087	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RIP, &u64Val);
5088	AssertRC(rc);
5089
5090	pCtx->rip = u64Val;
5091	EMHistoryUpdatePC(pVCpu, pCtx->rip, false);
5092	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
5093	}
5094	}
5095
5096
5097	/**
5098	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
5099	*
5100	* @param pVCpu The cross context virtual CPU structure.
5101	* @param pVmcsInfo The VMCS info. object.
5102	*
5103	* @remarks Called with interrupts and/or preemption disabled, should not assert!
5104	* @remarks Do -not- call this function directly, use vmxHCImportGuestState()
5105	* instead!!!
5106	*/
5107	static void vmxHCImportGuestRFlags(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
5108	{
5109	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5110	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
5111	{
5112	uint64_t u64Val;
5113	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RFLAGS, &u64Val);
5114	AssertRC(rc);
5115
5116	pCtx->rflags.u64 = u64Val;
5117	#ifdef IN_RING0
5118	PCVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
5119	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
5120	{
5121	pCtx->eflags.Bits.u1VM = 0;
5122	pCtx->eflags.Bits.u2IOPL = pVmcsInfoShared->RealMode.Eflags.Bits.u2IOPL;
5123	}
5124	#else
5125	RT_NOREF(pVmcsInfo);
5126	#endif
5127	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
5128	}
5129	}
5130
5131
5132	/**
5133	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
5134	* context.
5135	*
5136	* @param pVCpu The cross context virtual CPU structure.
5137	* @param pVmcsInfo The VMCS info. object.
5138	*
5139	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
5140	* do not log!
5141	* @remarks Do -not- call this function directly, use vmxHCImportGuestState()
5142	* instead!!!
5143	*/
5144	static void vmxHCImportGuestIntrState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
5145	{
5146	uint32_t u32Val;
5147	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &u32Val); AssertRC(rc);
5148	if (!u32Val)
5149	{
5150	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5151	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5152	CPUMSetGuestNmiBlocking(pVCpu, false);
5153	}
5154	else
5155	{
5156	/*
5157	* We must import RIP here to set our EM interrupt-inhibited state.
5158	* We also import RFLAGS as our code that evaluates pending interrupts
5159	* before VM-entry requires it.
5160	*/
5161	vmxHCImportGuestRip(pVCpu);
5162	vmxHCImportGuestRFlags(pVCpu, pVmcsInfo);
5163
5164	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
5165	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
5166	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5167	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5168
5169	bool const fNmiBlocking = RT_BOOL(u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
5170	CPUMSetGuestNmiBlocking(pVCpu, fNmiBlocking);
5171	}
5172	}
5173
5174
5175	/**
5176	* Worker for VMXR0ImportStateOnDemand.
5177	*
5178	* @returns VBox status code.
5179	* @param pVCpu The cross context virtual CPU structure.
5180	* @param pVmcsInfo The VMCS info. object.
5181	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
5182	*/
5183	static int vmxHCImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
5184	{
5185	int rc = VINF_SUCCESS;
5186	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5187	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5188	uint32_t u32Val;
5189
5190	/*
5191	* Note! This is hack to workaround a mysterious BSOD observed with release builds
5192	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
5193	* neither are other host platforms.
5194	*
5195	* Committing this temporarily as it prevents BSOD.
5196	*
5197	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
5198	*/
5199	# ifdef RT_OS_WINDOWS
5200	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
5201	return VERR_HM_IPE_1;
5202	# endif
5203
5204	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x);
5205
5206	#ifdef IN_RING0
5207	/*
5208	* We disable interrupts to make the updating of the state and in particular
5209	* the fExtrn modification atomic wrt to preemption hooks.
5210	*/
5211	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
5212	#endif
5213
5214	fWhat &= pCtx->fExtrn;
5215	if (fWhat)
5216	{
5217	do
5218	{
5219	if (fWhat & CPUMCTX_EXTRN_RIP)
5220	vmxHCImportGuestRip(pVCpu);
5221
5222	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
5223	vmxHCImportGuestRFlags(pVCpu, pVmcsInfo);
5224
5225	if (fWhat & (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI))
5226	vmxHCImportGuestIntrState(pVCpu, pVmcsInfo);
5227
5228	if (fWhat & CPUMCTX_EXTRN_RSP)
5229	{
5230	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RSP, &pCtx->rsp);
5231	AssertRC(rc);
5232	}
5233
5234	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
5235	{
5236	PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
5237	#ifndef IN_NEM_DARWIN
5238	bool const fRealOnV86Active = pVmcsInfoShared->RealMode.fRealOnV86Active;
5239	#else
5240	bool const fRealOnV86Active = false; /* HV supports only unrestricted guest execution. */
5241	#endif
5242	if (fWhat & CPUMCTX_EXTRN_CS)
5243	{
5244	vmxHCImportGuestSegReg(pVCpu, X86_SREG_CS);
5245	vmxHCImportGuestRip(pVCpu);
5246	if (fRealOnV86Active)
5247	pCtx->cs.Attr.u = pVmcsInfoShared->RealMode.AttrCS.u;
5248	EMHistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
5249	}
5250	if (fWhat & CPUMCTX_EXTRN_SS)
5251	{
5252	vmxHCImportGuestSegReg(pVCpu, X86_SREG_SS);
5253	if (fRealOnV86Active)
5254	pCtx->ss.Attr.u = pVmcsInfoShared->RealMode.AttrSS.u;
5255	}
5256	if (fWhat & CPUMCTX_EXTRN_DS)
5257	{
5258	vmxHCImportGuestSegReg(pVCpu, X86_SREG_DS);
5259	if (fRealOnV86Active)
5260	pCtx->ds.Attr.u = pVmcsInfoShared->RealMode.AttrDS.u;
5261	}
5262	if (fWhat & CPUMCTX_EXTRN_ES)
5263	{
5264	vmxHCImportGuestSegReg(pVCpu, X86_SREG_ES);
5265	if (fRealOnV86Active)
5266	pCtx->es.Attr.u = pVmcsInfoShared->RealMode.AttrES.u;
5267	}
5268	if (fWhat & CPUMCTX_EXTRN_FS)
5269	{
5270	vmxHCImportGuestSegReg(pVCpu, X86_SREG_FS);
5271	if (fRealOnV86Active)
5272	pCtx->fs.Attr.u = pVmcsInfoShared->RealMode.AttrFS.u;
5273	}
5274	if (fWhat & CPUMCTX_EXTRN_GS)
5275	{
5276	vmxHCImportGuestSegReg(pVCpu, X86_SREG_GS);
5277	if (fRealOnV86Active)
5278	pCtx->gs.Attr.u = pVmcsInfoShared->RealMode.AttrGS.u;
5279	}
5280	}
5281
5282	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
5283	{
5284	if (fWhat & CPUMCTX_EXTRN_LDTR)
5285	vmxHCImportGuestLdtr(pVCpu);
5286
5287	if (fWhat & CPUMCTX_EXTRN_GDTR)
5288	{
5289	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE, &pCtx->gdtr.pGdt); AssertRC(rc);
5290	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRC(rc);
5291	pCtx->gdtr.cbGdt = u32Val;
5292	}
5293
5294	/* Guest IDTR. */
5295	if (fWhat & CPUMCTX_EXTRN_IDTR)
5296	{
5297	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE, &pCtx->idtr.pIdt); AssertRC(rc);
5298	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRC(rc);
5299	pCtx->idtr.cbIdt = u32Val;
5300	}
5301
5302	/* Guest TR. */
5303	if (fWhat & CPUMCTX_EXTRN_TR)
5304	{
5305	#ifdef IN_RING0
5306	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
5307	don't need to import that one. */
5308	if (!pVmcsInfo->pShared->RealMode.fRealOnV86Active)
5309	#endif
5310	vmxHCImportGuestTr(pVCpu);
5311	}
5312	}
5313
5314	if (fWhat & CPUMCTX_EXTRN_DR7)
5315	{
5316	#ifdef IN_RING0
5317	if (!pVCpu->hmr0.s.fUsingHyperDR7)
5318	#endif
5319	{
5320	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_DR7, &pCtx->dr[7]);
5321	AssertRC(rc);
5322	}
5323	}
5324
5325	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
5326	{
5327	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip); AssertRC(rc);
5328	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp); AssertRC(rc);
5329	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRC(rc);
5330	pCtx->SysEnter.cs = u32Val;
5331	}
5332
5333	#ifdef IN_RING0
5334	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
5335	{
5336	if ( pVM->hmr0.s.fAllow64BitGuests
5337	&& (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
5338	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
5339	}
5340
5341	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
5342	{
5343	if ( pVM->hmr0.s.fAllow64BitGuests
5344	&& (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
5345	{
5346	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
5347	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
5348	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
5349	}
5350	}
5351
5352	if (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
5353	{
5354	PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
5355	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
5356	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
5357	Assert(pMsrs);
5358	Assert(cMsrs <= VMX_MISC_MAX_MSRS(g_HmMsrs.u.vmx.u64Misc));
5359	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
5360	for (uint32_t i = 0; i < cMsrs; i++)
5361	{
5362	uint32_t const idMsr = pMsrs[i].u32Msr;
5363	switch (idMsr)
5364	{
5365	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
5366	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
5367	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
5368	default:
5369	{
5370	uint32_t idxLbrMsr;
5371	if (VM_IS_VMX_LBR(pVM))
5372	{
5373	if (vmxHCIsLbrBranchFromMsr(pVM, idMsr, &idxLbrMsr))
5374	{
5375	Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr));
5376	pVmcsInfoShared->au64LbrFromIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
5377	break;
5378	}
5379	if (vmxHCIsLbrBranchToMsr(pVM, idMsr, &idxLbrMsr))
5380	{
5381	Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr));
5382	pVmcsInfoShared->au64LbrToIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
5383	break;
5384	}
5385	if (idMsr == pVM->hmr0.s.vmx.idLbrTosMsr)
5386	{
5387	pVmcsInfoShared->u64LbrTosMsr = pMsrs[i].u64Value;
5388	break;
5389	}
5390	/* Fallthru (no break) */
5391	}
5392	pCtx->fExtrn = 0;
5393	VCPU_2_VMXSTATE(pVCpu).u32HMError = pMsrs->u32Msr;
5394	ASMSetFlags(fEFlags);
5395	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
5396	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5397	}
5398	}
5399	}
5400	}
5401	#endif
5402
5403	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
5404	{
5405	if (fWhat & CPUMCTX_EXTRN_CR0)
5406	{
5407	uint64_t u64Cr0;
5408	uint64_t u64Shadow;
5409	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0, &u64Cr0); AssertRC(rc);
5410	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow); AssertRC(rc);
5411	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
5412	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
5413	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
5414	#else
5415	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
5416	{
5417	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
5418	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
5419	}
5420	else
5421	{
5422	/*
5423	* We've merged the guest and nested-guest's CR0 guest/host mask while executing
5424	* the nested-guest using hardware-assisted VMX. Accordingly we need to
5425	* re-construct CR0. See @bugref{9180#c95} for details.
5426	*/
5427	PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo;
5428	PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
5429	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
5430	\| (pVmcsNstGst->u64GuestCr0.u & pVmcsNstGst->u64Cr0Mask.u)
5431	\| (u64Shadow & (pVmcsInfoGst->u64Cr0Mask & ~pVmcsNstGst->u64Cr0Mask.u));
5432	}
5433	#endif
5434	#ifdef IN_RING0
5435	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
5436	#endif
5437	CPUMSetGuestCR0(pVCpu, u64Cr0);
5438	#ifdef IN_RING0
5439	VMMRZCallRing3Enable(pVCpu);
5440	#endif
5441	}
5442
5443	if (fWhat & CPUMCTX_EXTRN_CR4)
5444	{
5445	uint64_t u64Cr4;
5446	uint64_t u64Shadow;
5447	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR4, &u64Cr4); AssertRC(rc);
5448	rc \|= VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow); AssertRC(rc);
5449	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
5450	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
5451	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
5452	#else
5453	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
5454	{
5455	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
5456	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
5457	}
5458	else
5459	{
5460	/*
5461	* We've merged the guest and nested-guest's CR4 guest/host mask while executing
5462	* the nested-guest using hardware-assisted VMX. Accordingly we need to
5463	* re-construct CR4. See @bugref{9180#c95} for details.
5464	*/
5465	PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo;
5466	PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
5467	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
5468	\| (pVmcsNstGst->u64GuestCr4.u & pVmcsNstGst->u64Cr4Mask.u)
5469	\| (u64Shadow & (pVmcsInfoGst->u64Cr4Mask & ~pVmcsNstGst->u64Cr4Mask.u));
5470	}
5471	#endif
5472	pCtx->cr4 = u64Cr4;
5473	}
5474
5475	if (fWhat & CPUMCTX_EXTRN_CR3)
5476	{
5477	/* CR0.PG bit changes are always intercepted, so it's up to date. */
5478	if ( VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
5479	\|\| ( VM_IS_VMX_NESTED_PAGING(pVM)
5480	&& CPUMIsGuestPagingEnabledEx(pCtx)))
5481	{
5482	uint64_t u64Cr3;
5483	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR3, &u64Cr3); AssertRC(rc);
5484	if (pCtx->cr3 != u64Cr3)
5485	{
5486	pCtx->cr3 = u64Cr3;
5487	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5488	}
5489
5490	/*
5491	* If the guest is in PAE mode, sync back the PDPE's into the guest state.
5492	* CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date.
5493	*/
5494	if (CPUMIsGuestInPAEModeEx(pCtx))
5495	{
5496	X86PDPE aPaePdpes[4];
5497	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, &aPaePdpes[0].u); AssertRC(rc);
5498	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, &aPaePdpes[1].u); AssertRC(rc);
5499	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, &aPaePdpes[2].u); AssertRC(rc);
5500	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, &aPaePdpes[3].u); AssertRC(rc);
5501	if (memcmp(&aPaePdpes[0], &pCtx->aPaePdpes[0], sizeof(aPaePdpes)))
5502	{
5503	memcpy(&pCtx->aPaePdpes[0], &aPaePdpes[0], sizeof(aPaePdpes));
5504	/* PGM now updates PAE PDPTEs while updating CR3. */
5505	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5506	}
5507	}
5508	}
5509	}
5510	}
5511
5512	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5513	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
5514	{
5515	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
5516	&& !CPUMIsGuestInVmxNonRootMode(pCtx))
5517	{
5518	Assert(CPUMIsGuestInVmxRootMode(pCtx));
5519	rc = vmxHCCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
5520	if (RT_SUCCESS(rc))
5521	{ /* likely */ }
5522	else
5523	break;
5524	}
5525	}
5526	#endif
5527	} while (0);
5528
5529	if (RT_SUCCESS(rc))
5530	{
5531	/* Update fExtrn. */
5532	pCtx->fExtrn &= ~fWhat;
5533
5534	/* If everything has been imported, clear the HM keeper bit. */
5535	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
5536	{
5537	#ifndef IN_NEM_DARWIN
5538	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
5539	#else
5540	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_NEM;
5541	#endif
5542	Assert(!pCtx->fExtrn);
5543	}
5544	}
5545	}
5546	#ifdef IN_RING0
5547	else
5548	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
5549
5550	/*
5551	* Restore interrupts.
5552	*/
5553	ASMSetFlags(fEFlags);
5554	#endif
5555
5556	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x);
5557
5558	if (RT_SUCCESS(rc))
5559	{ /* likely */ }
5560	else
5561	return rc;
5562
5563	/*
5564	* Honor any pending CR3 updates.
5565	*
5566	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> VMXR0CallRing3Callback()
5567	* -> VMMRZCallRing3Disable() -> vmxHCImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5568	* -> continue with VM-exit handling -> vmxHCImportGuestState() and here we are.
5569	*
5570	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
5571	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
5572	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
5573	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
5574	*
5575	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5576	*
5577	* The force-flag is checked first as it's cheaper for potential superfluous calls to this function.
5578	*/
5579	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3)
5580	#ifdef IN_RING0
5581	&& VMMRZCallRing3IsEnabled(pVCpu)
5582	#endif
5583	)
5584	{
5585	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
5586	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5587	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5588	}
5589
5590	return VINF_SUCCESS;
5591	}
5592
5593
5594	/**
5595	* Check per-VM and per-VCPU force flag actions that require us to go back to
5596	* ring-3 for one reason or another.
5597	*
5598	* @returns Strict VBox status code (i.e. informational status codes too)
5599	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5600	* ring-3.
5601	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5602	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5603	* interrupts)
5604	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5605	* all EMTs to be in ring-3.
5606	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5607	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5608	* to the EM loop.
5609	*
5610	* @param pVCpu The cross context virtual CPU structure.
5611	* @param fIsNestedGuest Flag whether this is for a for a pending nested guest event.
5612	* @param fStepping Whether we are single-stepping the guest using the
5613	* hypervisor debugger.
5614	*
5615	* @remarks This might cause nested-guest VM-exits, caller must check if the guest
5616	* is no longer in VMX non-root mode.
5617	*/
5618	static VBOXSTRICTRC vmxHCCheckForceFlags(PVMCPUCC pVCpu, bool fIsNestedGuest, bool fStepping)
5619	{
5620	#ifdef IN_RING0
5621	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5622	#endif
5623
5624	/*
5625	* Update pending interrupts into the APIC's IRR.
5626	*/
5627	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
5628	APICUpdatePendingInterrupts(pVCpu);
5629
5630	/*
5631	* Anything pending? Should be more likely than not if we're doing a good job.
5632	*/
5633	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5634	if ( !fStepping
5635	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
5636	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
5637	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
5638	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
5639	return VINF_SUCCESS;
5640
5641	/* Pending PGM C3 sync. */
5642	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5643	{
5644	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5645	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
5646	VBOXSTRICTRC rcStrict = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
5647	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5648	if (rcStrict != VINF_SUCCESS)
5649	{
5650	AssertRC(VBOXSTRICTRC_VAL(rcStrict));
5651	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
5652	return rcStrict;
5653	}
5654	}
5655
5656	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5657	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
5658	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5659	{
5660	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchHmToR3FF);
5661	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
5662	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5663	return rc;
5664	}
5665
5666	/* Pending VM request packets, such as hardware interrupts. */
5667	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
5668	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
5669	{
5670	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchVmReq);
5671	Log4Func(("Pending VM request forcing us back to ring-3\n"));
5672	return VINF_EM_PENDING_REQUEST;
5673	}
5674
5675	/* Pending PGM pool flushes. */
5676	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5677	{
5678	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchPgmPoolFlush);
5679	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
5680	return VINF_PGM_POOL_FLUSH_PENDING;
5681	}
5682
5683	/* Pending DMA requests. */
5684	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
5685	{
5686	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchDma);
5687	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
5688	return VINF_EM_RAW_TO_R3;
5689	}
5690
5691	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5692	/*
5693	* Pending nested-guest events.
5694	*
5695	* Please note the priority of these events are specified and important.
5696	* See Intel spec. 29.4.3.2 "APIC-Write Emulation".
5697	* See Intel spec. 6.9 "Priority Among Simultaneous Exceptions And Interrupts".
5698	*/
5699	if (fIsNestedGuest)
5700	{
5701	/* Pending nested-guest APIC-write. */
5702	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
5703	{
5704	Log4Func(("Pending nested-guest APIC-write\n"));
5705	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitApicWrite(pVCpu);
5706	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
5707	return rcStrict;
5708	}
5709
5710	/* Pending nested-guest monitor-trap flag (MTF). */
5711	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF))
5712	{
5713	Log4Func(("Pending nested-guest MTF\n"));
5714	VBOXSTRICTRC rcStrict = IEMExecVmxVmexit(pVCpu, VMX_EXIT_MTF, 0 /* uExitQual */);
5715	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
5716	return rcStrict;
5717	}
5718
5719	/* Pending nested-guest VMX-preemption timer expired. */
5720	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER))
5721	{
5722	Log4Func(("Pending nested-guest preempt timer\n"));
5723	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitPreemptTimer(pVCpu);
5724	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
5725	return rcStrict;
5726	}
5727	}
5728	#else
5729	NOREF(fIsNestedGuest);
5730	#endif
5731
5732	return VINF_SUCCESS;
5733	}
5734
5735
5736	/**
5737	* Converts any TRPM trap into a pending HM event. This is typically used when
5738	* entering from ring-3 (not longjmp returns).
5739	*
5740	* @param pVCpu The cross context virtual CPU structure.
5741	*/
5742	static void vmxHCTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
5743	{
5744	Assert(TRPMHasTrap(pVCpu));
5745	Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending);
5746
5747	uint8_t uVector;
5748	TRPMEVENT enmTrpmEvent;
5749	uint32_t uErrCode;
5750	RTGCUINTPTR GCPtrFaultAddress;
5751	uint8_t cbInstr;
5752	bool fIcebp;
5753
5754	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, &fIcebp);
5755	AssertRC(rc);
5756
5757	uint32_t u32IntInfo;
5758	u32IntInfo = uVector \| VMX_IDT_VECTORING_INFO_VALID;
5759	u32IntInfo \|= HMTrpmEventTypeToVmxEventType(uVector, enmTrpmEvent, fIcebp);
5760
5761	rc = TRPMResetTrap(pVCpu);
5762	AssertRC(rc);
5763	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5764	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5765
5766	vmxHCSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5767	}
5768
5769
5770	/**
5771	* Converts the pending HM event into a TRPM trap.
5772	*
5773	* @param pVCpu The cross context virtual CPU structure.
5774	*/
5775	static void vmxHCPendingEventToTrpmTrap(PVMCPUCC pVCpu)
5776	{
5777	Assert(VCPU_2_VMXSTATE(pVCpu).Event.fPending);
5778
5779	/* If a trap was already pending, we did something wrong! */
5780	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5781
5782	uint32_t const u32IntInfo = VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo;
5783	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(u32IntInfo);
5784	TRPMEVENT const enmTrapType = HMVmxEventTypeToTrpmEventType(u32IntInfo);
5785
5786	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5787
5788	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5789	AssertRC(rc);
5790
5791	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
5792	TRPMSetErrorCode(pVCpu, VCPU_2_VMXSTATE(pVCpu).Event.u32ErrCode);
5793
5794	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(u32IntInfo))
5795	TRPMSetFaultAddress(pVCpu, VCPU_2_VMXSTATE(pVCpu).Event.GCPtrFaultAddress);
5796	else
5797	{
5798	uint8_t const uVectorType = VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo);
5799	switch (uVectorType)
5800	{
5801	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5802	TRPMSetTrapDueToIcebp(pVCpu);
5803	RT_FALL_THRU();
5804	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5805	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5806	{
5807	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5808	\|\| ( uVector == X86_XCPT_BP /* INT3 */
5809	\|\| uVector == X86_XCPT_OF /* INTO */
5810	\|\| uVector == X86_XCPT_DB /* INT1 (ICEBP) */),
5811	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5812	TRPMSetInstrLength(pVCpu, VCPU_2_VMXSTATE(pVCpu).Event.cbInstr);
5813	break;
5814	}
5815	}
5816	}
5817
5818	/* We're now done converting the pending event. */
5819	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false;
5820	}
5821
5822
5823	/**
5824	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
5825	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
5826	*
5827	* @param pVCpu The cross context virtual CPU structure.
5828	* @param pVmcsInfo The VMCS info. object.
5829	*/
5830	static void vmxHCSetIntWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
5831	{
5832	if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
5833	{
5834	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
5835	{
5836	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
5837	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
5838	AssertRC(rc);
5839	}
5840	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
5841	}
5842
5843
5844	/**
5845	* Clears the interrupt-window exiting control in the VMCS.
5846	*
5847	* @param pVCpu The cross context virtual CPU structure.
5848	* @param pVmcsInfo The VMCS info. object.
5849	*/
5850	DECLINLINE(void) vmxHCClearIntWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
5851	{
5852	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
5853	{
5854	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
5855	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
5856	AssertRC(rc);
5857	}
5858	}
5859
5860
5861	/**
5862	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
5863	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
5864	*
5865	* @param pVCpu The cross context virtual CPU structure.
5866	* @param pVmcsInfo The VMCS info. object.
5867	*/
5868	static void vmxHCSetNmiWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
5869	{
5870	if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
5871	{
5872	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
5873	{
5874	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
5875	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
5876	AssertRC(rc);
5877	Log4Func(("Setup NMI-window exiting\n"));
5878	}
5879	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
5880	}
5881
5882
5883	/**
5884	* Clears the NMI-window exiting control in the VMCS.
5885	*
5886	* @param pVCpu The cross context virtual CPU structure.
5887	* @param pVmcsInfo The VMCS info. object.
5888	*/
5889	DECLINLINE(void) vmxHCClearNmiWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
5890	{
5891	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
5892	{
5893	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
5894	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
5895	AssertRC(rc);
5896	}
5897	}
5898
5899
5900	#ifdef IN_RING0
5901	/**
5902	* Does the necessary state syncing before returning to ring-3 for any reason
5903	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
5904	*
5905	* @returns VBox status code.
5906	* @param pVCpu The cross context virtual CPU structure.
5907	* @param fImportState Whether to import the guest state from the VMCS back
5908	* to the guest-CPU context.
5909	*
5910	* @remarks No-long-jmp zone!!!
5911	*/
5912	static int vmxHCLeave(PVMCPUCC pVCpu, bool fImportState)
5913	{
5914	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5915	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5916
5917	RTCPUID const idCpu = RTMpCpuId();
5918	Log4Func(("HostCpuId=%u\n", idCpu));
5919
5920	/*
5921	* !!! IMPORTANT !!!
5922	* If you modify code here, check whether VMXR0CallRing3Callback() needs to be updated too.
5923	*/
5924
5925	/* Save the guest state if necessary. */
5926	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
5927	if (fImportState)
5928	{
5929	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
5930	AssertRCReturn(rc, rc);
5931	}
5932
5933	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
5934	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
5935	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5936
5937	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
5938	#ifdef VBOX_STRICT
5939	if (CPUMIsHyperDebugStateActive(pVCpu))
5940	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
5941	#endif
5942	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
5943	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5944	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5945
5946	/* Restore host-state bits that VT-x only restores partially. */
5947	if (pVCpu->hmr0.s.vmx.fRestoreHostFlags > VMX_RESTORE_HOST_REQUIRED)
5948	{
5949	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hmr0.s.vmx.fRestoreHostFlags, idCpu));
5950	VMXRestoreHostState(pVCpu->hmr0.s.vmx.fRestoreHostFlags, &pVCpu->hmr0.s.vmx.RestoreHost);
5951	}
5952	pVCpu->hmr0.s.vmx.fRestoreHostFlags = 0;
5953
5954	/* Restore the lazy host MSRs as we're leaving VT-x context. */
5955	if (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
5956	{
5957	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
5958	if (!fImportState)
5959	{
5960	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
5961	AssertRCReturn(rc, rc);
5962	}
5963	vmxHCLazyRestoreHostMsrs(pVCpu);
5964	Assert(!pVCpu->hmr0.s.vmx.fLazyMsrs);
5965	}
5966	else
5967	pVCpu->hmr0.s.vmx.fLazyMsrs = 0;
5968
5969	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
5970	pVCpu->hmr0.s.vmx.fUpdatedHostAutoMsrs = false;
5971
5972	STAM_PROFILE_ADV_SET_STOPPED(&VCPU_2_VMXSTATS(pVCpu).StatEntry);
5973	STAM_PROFILE_ADV_SET_STOPPED(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState);
5974	STAM_PROFILE_ADV_SET_STOPPED(&VCPU_2_VMXSTATS(pVCpu).StatExportGuestState);
5975	STAM_PROFILE_ADV_SET_STOPPED(&VCPU_2_VMXSTATS(pVCpu).StatPreExit);
5976	STAM_PROFILE_ADV_SET_STOPPED(&VCPU_2_VMXSTATS(pVCpu).StatExitHandling);
5977	STAM_PROFILE_ADV_SET_STOPPED(&VCPU_2_VMXSTATS(pVCpu).StatExitIO);
5978	STAM_PROFILE_ADV_SET_STOPPED(&VCPU_2_VMXSTATS(pVCpu).StatExitMovCRx);
5979	STAM_PROFILE_ADV_SET_STOPPED(&VCPU_2_VMXSTATS(pVCpu).StatExitXcptNmi);
5980	STAM_PROFILE_ADV_SET_STOPPED(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry);
5981	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchLongJmpToR3);
5982
5983	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
5984
5985	/** @todo This partially defeats the purpose of having preemption hooks.
5986	* The problem is, deregistering the hooks should be moved to a place that
5987	* lasts until the EMT is about to be destroyed not everytime while leaving HM
5988	* context.
5989	*/
5990	int rc = vmxHCClearVmcs(pVmcsInfo);
5991	AssertRCReturn(rc, rc);
5992
5993	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5994	/*
5995	* A valid shadow VMCS is made active as part of VM-entry. It is necessary to
5996	* clear a shadow VMCS before allowing that VMCS to become active on another
5997	* logical processor. We may or may not be importing guest state which clears
5998	* it, so cover for it here.
5999	*
6000	* See Intel spec. 24.11.1 "Software Use of Virtual-Machine Control Structures".
6001	*/
6002	if ( pVmcsInfo->pvShadowVmcs
6003	&& pVmcsInfo->fShadowVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
6004	{
6005	rc = vmxHCClearShadowVmcs(pVmcsInfo);
6006	AssertRCReturn(rc, rc);
6007	}
6008
6009	/*
6010	* Flag that we need to re-export the host state if we switch to this VMCS before
6011	* executing guest or nested-guest code.
6012	*/
6013	pVmcsInfo->idHostCpuState = NIL_RTCPUID;
6014	#endif
6015
6016	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6017	NOREF(idCpu);
6018	return VINF_SUCCESS;
6019	}
6020
6021
6022	/**
6023	* Leaves the VT-x session.
6024	*
6025	* @returns VBox status code.
6026	* @param pVCpu The cross context virtual CPU structure.
6027	*
6028	* @remarks No-long-jmp zone!!!
6029	*/
6030	static int vmxHCLeaveSession(PVMCPUCC pVCpu)
6031	{
6032	HM_DISABLE_PREEMPT(pVCpu);
6033	HMVMX_ASSERT_CPU_SAFE(pVCpu);
6034	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6035	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6036
6037	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6038	and done this from the VMXR0ThreadCtxCallback(). */
6039	if (!pVCpu->hmr0.s.fLeaveDone)
6040	{
6041	int rc2 = vmxHCLeave(pVCpu, true /* fImportState */);
6042	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
6043	pVCpu->hmr0.s.fLeaveDone = true;
6044	}
6045	Assert(!pVCpu->cpum.GstCtx.fExtrn);
6046
6047	/*
6048	* !!! IMPORTANT !!!
6049	* If you modify code here, make sure to check whether VMXR0CallRing3Callback() needs to be updated too.
6050	*/
6051
6052	/* Deregister hook now that we've left HM context before re-enabling preemption. */
6053	/** @todo Deregistering here means we need to VMCLEAR always
6054	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
6055	* for calling VMMR0ThreadCtxHookDisable here! */
6056	VMMR0ThreadCtxHookDisable(pVCpu);
6057
6058	/* Leave HM context. This takes care of local init (term) and deregistering the longjmp-to-ring-3 callback. */
6059	int rc = HMR0LeaveCpu(pVCpu);
6060	HM_RESTORE_PREEMPT();
6061	return rc;
6062	}
6063
6064
6065	/**
6066	* Does the necessary state syncing before doing a longjmp to ring-3.
6067	*
6068	* @returns VBox status code.
6069	* @param pVCpu The cross context virtual CPU structure.
6070	*
6071	* @remarks No-long-jmp zone!!!
6072	*/
6073	DECLINLINE(int) vmxHCLongJmpToRing3(PVMCPUCC pVCpu)
6074	{
6075	return vmxHCLeaveSession(pVCpu);
6076	}
6077
6078
6079	/**
6080	* Take necessary actions before going back to ring-3.
6081	*
6082	* An action requires us to go back to ring-3. This function does the necessary
6083	* steps before we can safely return to ring-3. This is not the same as longjmps
6084	* to ring-3, this is voluntary and prepares the guest so it may continue
6085	* executing outside HM (recompiler/IEM).
6086	*
6087	* @returns VBox status code.
6088	* @param pVCpu The cross context virtual CPU structure.
6089	* @param rcExit The reason for exiting to ring-3. Can be
6090	* VINF_VMM_UNKNOWN_RING3_CALL.
6091	*/
6092	static int vmxHCExitToRing3(PVMCPUCC pVCpu, VBOXSTRICTRC rcExit)
6093	{
6094	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6095
6096	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6097	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6098	{
6099	VMXGetCurrentVmcs(&VCPU_2_VMXSTATE(pVCpu).vmx.LastError.HCPhysCurrentVmcs);
6100	VCPU_2_VMXSTATE(pVCpu).vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
6101	VCPU_2_VMXSTATE(pVCpu).vmx.LastError.idEnteredCpu = pVCpu->hmr0.s.idEnteredCpu;
6102	/* LastError.idCurrentCpu was updated in vmxHCPreRunGuestCommitted(). */
6103	}
6104
6105	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6106	VMMRZCallRing3Disable(pVCpu);
6107	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
6108
6109	/*
6110	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
6111	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
6112	*
6113	* This is because execution may continue from ring-3 and we would need to inject
6114	* the event from there (hence place it back in TRPM).
6115	*/
6116	if (VCPU_2_VMXSTATE(pVCpu).Event.fPending)
6117	{
6118	vmxHCPendingEventToTrpmTrap(pVCpu);
6119	Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending);
6120
6121	/* Clear the events from the VMCS. */
6122	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6123	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, 0); AssertRC(rc);
6124	}
6125	#ifdef VBOX_STRICT
6126	/*
6127	* We check for rcExit here since for errors like VERR_VMX_UNABLE_TO_START_VM (which are
6128	* fatal), we don't care about verifying duplicate injection of events. Errors like
6129	* VERR_EM_INTERPRET are converted to their VINF_* counterparts -prior- to calling this
6130	* function so those should and will be checked below.
6131	*/
6132	else if (RT_SUCCESS(rcExit))
6133	{
6134	/*
6135	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
6136	* This can be pretty hard to debug otherwise, interrupts might get injected twice
6137	* occasionally, see @bugref{9180#c42}.
6138	*
6139	* However, if the VM-entry failed, any VM entry-interruption info. field would
6140	* be left unmodified as the event would not have been injected to the guest. In
6141	* such cases, don't assert, we're not going to continue guest execution anyway.
6142	*/
6143	uint32_t uExitReason;
6144	uint32_t uEntryIntInfo;
6145	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6146	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
6147	AssertRC(rc);
6148	AssertMsg(VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason) \|\| !VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo),
6149	("uExitReason=%#RX32 uEntryIntInfo=%#RX32 rcExit=%d\n", uExitReason, uEntryIntInfo, VBOXSTRICTRC_VAL(rcExit)));
6150	}
6151	#endif
6152
6153	/*
6154	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
6155	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
6156	* (e.g. TPR below threshold).
6157	*/
6158	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
6159	{
6160	vmxHCClearIntWindowExitVmcs(pVCpu, pVmcsInfo);
6161	vmxHCClearNmiWindowExitVmcs(pVCpu, pVmcsInfo);
6162	}
6163
6164	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
6165	and if we're injecting an event we should have a TRPM trap pending. */
6166	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6167	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
6168	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6169	#endif
6170
6171	/* Save guest state and restore host state bits. */
6172	int rc = vmxHCLeaveSession(pVCpu);
6173	AssertRCReturn(rc, rc);
6174	STAM_COUNTER_DEC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchLongJmpToR3);
6175
6176	/* Thread-context hooks are unregistered at this point!!! */
6177	/* Ring-3 callback notifications are unregistered at this point!!! */
6178
6179	/* Sync recompiler state. */
6180	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6181	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6182	\| CPUM_CHANGED_LDTR
6183	\| CPUM_CHANGED_GDTR
6184	\| CPUM_CHANGED_IDTR
6185	\| CPUM_CHANGED_TR
6186	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6187	if ( pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging
6188	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
6189	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6190
6191	Assert(!pVCpu->hmr0.s.fClearTrapFlag);
6192
6193	/* Update the exit-to-ring 3 reason. */
6194	VCPU_2_VMXSTATE(pVCpu).rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
6195
6196	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6197	if ( rcExit != VINF_EM_RAW_INTERRUPT
6198	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
6199	{
6200	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
6201	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
6202	}
6203
6204	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchExitToR3);
6205	VMMRZCallRing3Enable(pVCpu);
6206	return rc;
6207	}
6208
6209
6210	/**
6211	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6212	* stack.
6213	*
6214	* @returns Strict VBox status code (i.e. informational status codes too).
6215	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6216	* @param pVCpu The cross context virtual CPU structure.
6217	* @param uValue The value to push to the guest stack.
6218	*/
6219	static VBOXSTRICTRC vmxHCRealModeGuestStackPush(PVMCPUCC pVCpu, uint16_t uValue)
6220	{
6221	/*
6222	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6223	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6224	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6225	*/
6226	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6227	if (pCtx->sp == 1)
6228	return VINF_EM_RESET;
6229	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6230	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
6231	AssertRC(rc);
6232	return rc;
6233	}
6234	#endif /* !IN_RING */
6235
6236	/**
6237	* Injects an event into the guest upon VM-entry by updating the relevant fields
6238	* in the VM-entry area in the VMCS.
6239	*
6240	* @returns Strict VBox status code (i.e. informational status codes too).
6241	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6242	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6243	*
6244	* @param pVCpu The cross context virtual CPU structure.
6245	* @param pVmxTransient The VMX-transient structure.
6246	* @param pEvent The event being injected.
6247	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state. This
6248	* will be updated if necessary. This cannot not be NULL.
6249	* @param fStepping Whether we're single-stepping guest execution and should
6250	* return VINF_EM_DBG_STEPPED if the event is injected
6251	* directly (registers modified by us, not by hardware on
6252	* VM-entry).
6253	*/
6254	static VBOXSTRICTRC vmxHCInjectEventVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNestedGuest, PCHMEVENT pEvent, bool fStepping,
6255	uint32_t *pfIntrState)
6256	{
6257	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6258	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
6259	Assert(pfIntrState);
6260
6261	#ifndef IN_RING0
6262	RT_NOREF(fIsNestedGuest, fStepping, pfIntrState);
6263	#endif
6264
6265	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6266	uint32_t u32IntInfo = pEvent->u64IntInfo;
6267	uint32_t const u32ErrCode = pEvent->u32ErrCode;
6268	uint32_t const cbInstr = pEvent->cbInstr;
6269	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
6270	uint8_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
6271	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
6272
6273	#ifdef VBOX_STRICT
6274	/*
6275	* Validate the error-code-valid bit for hardware exceptions.
6276	* No error codes for exceptions in real-mode.
6277	*
6278	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
6279	*/
6280	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
6281	&& !CPUMIsGuestInRealModeEx(pCtx))
6282	{
6283	switch (uVector)
6284	{
6285	case X86_XCPT_PF:
6286	case X86_XCPT_DF:
6287	case X86_XCPT_TS:
6288	case X86_XCPT_NP:
6289	case X86_XCPT_SS:
6290	case X86_XCPT_GP:
6291	case X86_XCPT_AC:
6292	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
6293	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
6294	RT_FALL_THRU();
6295	default:
6296	break;
6297	}
6298	}
6299
6300	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6301	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
6302	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
6303	#endif
6304
6305	RT_NOREF(uVector);
6306	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
6307	\|\| uIntType == VMX_EXIT_INT_INFO_TYPE_NMI
6308	\|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT
6309	\|\| uIntType == VMX_EXIT_INT_INFO_TYPE_SW_XCPT)
6310	{
6311	Assert(uVector <= X86_XCPT_LAST);
6312	Assert(uIntType != VMX_EXIT_INT_INFO_TYPE_NMI \|\| uVector == X86_XCPT_NMI);
6313	Assert(uIntType != VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT \|\| uVector == X86_XCPT_DB);
6314	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).aStatInjectedXcpts[uVector]);
6315	}
6316	else
6317	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).aStatInjectedIrqs[uVector & MASK_INJECT_IRQ_STAT]);
6318
6319	/*
6320	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
6321	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
6322	* interrupt handler in the (real-mode) guest.
6323	*
6324	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
6325	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6326	*/
6327	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
6328	{
6329	#ifdef IN_RING0
6330	if (pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fUnrestrictedGuest)
6331	#endif
6332	{
6333	/*
6334	* For CPUs with unrestricted guest execution enabled and with the guest
6335	* in real-mode, we must not set the deliver-error-code bit.
6336	*
6337	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6338	*/
6339	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
6340	}
6341	#ifdef IN_RING0
6342	else
6343	{
6344	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6345	Assert(PDMVmmDevHeapIsEnabled(pVM));
6346	Assert(pVM->hm.s.vmx.pRealModeTSS);
6347	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
6348
6349	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
6350	int rc2 = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
6351	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
6352	AssertRCReturn(rc2, rc2);
6353
6354	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6355	size_t const cbIdtEntry = sizeof(X86IDTR16);
6356	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
6357	{
6358	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6359	if (uVector == X86_XCPT_DF)
6360	return VINF_EM_RESET;
6361
6362	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
6363	No error codes for exceptions in real-mode. */
6364	if (uVector == X86_XCPT_GP)
6365	{
6366	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
6367	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
6368	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6369	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6370	HMEVENT EventXcptDf;
6371	RT_ZERO(EventXcptDf);
6372	EventXcptDf.u64IntInfo = uXcptDfInfo;
6373	return vmxHCInjectEventVmcs(pVCpu, pVmcsInfo, fIsNestedGuest, &EventXcptDf, fStepping, pfIntrState);
6374	}
6375
6376	/*
6377	* If we're injecting an event with no valid IDT entry, inject a #GP.
6378	* No error codes for exceptions in real-mode.
6379	*
6380	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
6381	*/
6382	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
6383	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
6384	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6385	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6386	HMEVENT EventXcptGp;
6387	RT_ZERO(EventXcptGp);
6388	EventXcptGp.u64IntInfo = uXcptGpInfo;
6389	return vmxHCInjectEventVmcs(pVCpu, pVmcsInfo, fIsNestedGuest, &EventXcptGp, fStepping, pfIntrState);
6390	}
6391
6392	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6393	uint16_t uGuestIp = pCtx->ip;
6394	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
6395	{
6396	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6397	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6398	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
6399	}
6400	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
6401	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
6402
6403	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6404	X86IDTR16 IdtEntry;
6405	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
6406	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
6407	AssertRCReturn(rc2, rc2);
6408
6409	/* Construct the stack frame for the interrupt/exception handler. */
6410	VBOXSTRICTRC rcStrict;
6411	rcStrict = vmxHCRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
6412	if (rcStrict == VINF_SUCCESS)
6413	{
6414	rcStrict = vmxHCRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
6415	if (rcStrict == VINF_SUCCESS)
6416	rcStrict = vmxHCRealModeGuestStackPush(pVCpu, uGuestIp);
6417	}
6418
6419	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6420	if (rcStrict == VINF_SUCCESS)
6421	{
6422	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6423	pCtx->rip = IdtEntry.offSel;
6424	pCtx->cs.Sel = IdtEntry.uSel;
6425	pCtx->cs.ValidSel = IdtEntry.uSel;
6426	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
6427	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
6428	&& uVector == X86_XCPT_PF)
6429	pCtx->cr2 = GCPtrFault;
6430
6431	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
6432	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
6433	\| HM_CHANGED_GUEST_RSP);
6434
6435	/*
6436	* If we delivered a hardware exception (other than an NMI) and if there was
6437	* block-by-STI in effect, we should clear it.
6438	*/
6439	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
6440	{
6441	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
6442	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
6443	Log4Func(("Clearing inhibition due to STI\n"));
6444	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
6445	}
6446
6447	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
6448	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
6449
6450	/*
6451	* The event has been truly dispatched to the guest. Mark it as no longer pending so
6452	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
6453	*/
6454	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false;
6455
6456	/*
6457	* If we eventually support nested-guest execution without unrestricted guest execution,
6458	* we should set fInterceptEvents here.
6459	*/
6460	Assert(!fIsNestedGuest);
6461
6462	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
6463	if (fStepping)
6464	rcStrict = VINF_EM_DBG_STEPPED;
6465	}
6466	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
6467	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6468	return rcStrict;
6469	}
6470	#else
6471	RT_NOREF(pVmcsInfo);
6472	#endif
6473	}
6474
6475	/*
6476	* Validate.
6477	*/
6478	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6479	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
6480
6481	/*
6482	* Inject the event into the VMCS.
6483	*/
6484	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
6485	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
6486	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6487	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6488	AssertRC(rc);
6489
6490	/*
6491	* Update guest CR2 if this is a page-fault.
6492	*/
6493	if (VMX_ENTRY_INT_INFO_IS_XCPT_PF(u32IntInfo))
6494	pCtx->cr2 = GCPtrFault;
6495
6496	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
6497	return VINF_SUCCESS;
6498	}
6499
6500
6501	/**
6502	* Evaluates the event to be delivered to the guest and sets it as the pending
6503	* event.
6504	*
6505	* Toggling of interrupt force-flags here is safe since we update TRPM on premature
6506	* exits to ring-3 before executing guest code, see vmxHCExitToRing3(). We must
6507	* NOT restore these force-flags.
6508	*
6509	* @returns Strict VBox status code (i.e. informational status codes too).
6510	* @param pVCpu The cross context virtual CPU structure.
6511	* @param pVmcsInfo The VMCS information structure.
6512	* @param fIsNestedGuest Flag whether the evaluation happens for a nestd guest.
6513	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
6514	*/
6515	static VBOXSTRICTRC vmxHCEvaluatePendingEvent(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNestedGuest, uint32_t *pfIntrState)
6516	{
6517	Assert(pfIntrState);
6518	Assert(!TRPMHasTrap(pVCpu));
6519
6520	/*
6521	* Compute/update guest-interruptibility state related FFs.
6522	* The FFs will be used below while evaluating events to be injected.
6523	*/
6524	*pfIntrState = vmxHCGetGuestIntrStateAndUpdateFFs(pVCpu);
6525
6526	/*
6527	* Evaluate if a new event needs to be injected.
6528	* An event that's already pending has already performed all necessary checks.
6529	*/
6530	if ( !VCPU_2_VMXSTATE(pVCpu).Event.fPending
6531	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6532	{
6533	/** @todo SMI. SMIs take priority over NMIs. */
6534
6535	/*
6536	* NMIs.
6537	* NMIs take priority over external interrupts.
6538	*/
6539	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6540	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6541	#endif
6542	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
6543	{
6544	/*
6545	* For a guest, the FF always indicates the guest's ability to receive an NMI.
6546	*
6547	* For a nested-guest, the FF always indicates the outer guest's ability to
6548	* receive an NMI while the guest-interruptibility state bit depends on whether
6549	* the nested-hypervisor is using virtual-NMIs.
6550	*/
6551	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
6552	{
6553	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6554	if ( fIsNestedGuest
6555	&& CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_NMI_EXIT))
6556	return IEMExecVmxVmexitXcptNmi(pVCpu);
6557	#endif
6558	vmxHCSetPendingXcptNmi(pVCpu);
6559	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6560	Log4Func(("NMI pending injection\n"));
6561
6562	/* We've injected the NMI, bail. */
6563	return VINF_SUCCESS;
6564	}
6565	else if (!fIsNestedGuest)
6566	vmxHCSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
6567	}
6568
6569	/*
6570	* External interrupts (PIC/APIC).
6571	* Once PDMGetInterrupt() returns a valid interrupt we -must- deliver it.
6572	* We cannot re-request the interrupt from the controller again.
6573	*/
6574	if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
6575	&& !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
6576	{
6577	Assert(!DBGFIsStepping(pVCpu));
6578	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
6579	AssertRC(rc);
6580
6581	/*
6582	* We must not check EFLAGS directly when executing a nested-guest, use
6583	* CPUMIsGuestPhysIntrEnabled() instead as EFLAGS.IF does not control the blocking of
6584	* external interrupts when "External interrupt exiting" is set. This fixes a nasty
6585	* SMP hang while executing nested-guest VCPUs on spinlocks which aren't rescued by
6586	* other VM-exits (like a preemption timer), see @bugref{9562#c18}.
6587	*
6588	* See Intel spec. 25.4.1 "Event Blocking".
6589	*/
6590	if (CPUMIsGuestPhysIntrEnabled(pVCpu))
6591	{
6592	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6593	if ( fIsNestedGuest
6594	&& CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
6595	{
6596	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0 /* uVector /, true / fIntPending */);
6597	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
6598	return rcStrict;
6599	}
6600	#endif
6601	uint8_t u8Interrupt;
6602	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6603	if (RT_SUCCESS(rc))
6604	{
6605	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6606	if ( fIsNestedGuest
6607	&& CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
6608	{
6609	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
6610	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
6611	return rcStrict;
6612	}
6613	#endif
6614	vmxHCSetPendingExtInt(pVCpu, u8Interrupt);
6615	Log4Func(("External interrupt (%#x) pending injection\n", u8Interrupt));
6616	}
6617	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
6618	{
6619	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchTprMaskedIrq);
6620
6621	if ( !fIsNestedGuest
6622	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
6623	vmxHCApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
6624	/* else: for nested-guests, TPR threshold is picked up while merging VMCS controls. */
6625
6626	/*
6627	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
6628	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
6629	* need to re-set this force-flag here.
6630	*/
6631	}
6632	else
6633	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchGuestIrq);
6634
6635	/* We've injected the interrupt or taken necessary action, bail. */
6636	return VINF_SUCCESS;
6637	}
6638	if (!fIsNestedGuest)
6639	vmxHCSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
6640	}
6641	}
6642	else if (!fIsNestedGuest)
6643	{
6644	/*
6645	* An event is being injected or we are in an interrupt shadow. Check if another event is
6646	* pending. If so, instruct VT-x to cause a VM-exit as soon as the guest is ready to accept
6647	* the pending event.
6648	*/
6649	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
6650	vmxHCSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
6651	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
6652	&& !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
6653	vmxHCSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
6654	}
6655	/* else: for nested-guests, NMI/interrupt-window exiting will be picked up when merging VMCS controls. */
6656
6657	return VINF_SUCCESS;
6658	}
6659
6660
6661	/**
6662	* Injects any pending events into the guest if the guest is in a state to
6663	* receive them.
6664	*
6665	* @returns Strict VBox status code (i.e. informational status codes too).
6666	* @param pVCpu The cross context virtual CPU structure.
6667	* @param fIsNestedGuest Flag whether the event injection happens for a nested guest.
6668	* @param fIntrState The VT-x guest-interruptibility state.
6669	* @param fStepping Whether we are single-stepping the guest using the
6670	* hypervisor debugger and should return
6671	* VINF_EM_DBG_STEPPED if the event was dispatched
6672	* directly.
6673	*/
6674	static VBOXSTRICTRC vmxHCInjectPendingEvent(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNestedGuest, uint32_t fIntrState, bool fStepping)
6675	{
6676	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6677	#ifdef IN_RING0
6678	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6679	#endif
6680
6681	#ifdef VBOX_STRICT
6682	/*
6683	* Verify guest-interruptibility state.
6684	*
6685	* We put this in a scoped block so we do not accidentally use fBlockSti or fBlockMovSS,
6686	* since injecting an event may modify the interruptibility state and we must thus always
6687	* use fIntrState.
6688	*/
6689	{
6690	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
6691	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
6692	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
6693	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6694	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
6695	Assert(!TRPMHasTrap(pVCpu));
6696	NOREF(fBlockMovSS); NOREF(fBlockSti);
6697	}
6698	#endif
6699
6700	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
6701	if (VCPU_2_VMXSTATE(pVCpu).Event.fPending)
6702	{
6703	/*
6704	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
6705	* pending even while injecting an event and in this case, we want a VM-exit as soon as
6706	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
6707	*
6708	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
6709	*/
6710	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo);
6711	#ifdef VBOX_STRICT
6712	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6713	{
6714	Assert(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
6715	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI));
6716	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
6717	}
6718	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
6719	{
6720	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI));
6721	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI));
6722	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
6723	}
6724	#endif
6725	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
6726	uIntType));
6727
6728	/*
6729	* Inject the event and get any changes to the guest-interruptibility state.
6730	*
6731	* The guest-interruptibility state may need to be updated if we inject the event
6732	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
6733	*/
6734	rcStrict = vmxHCInjectEventVmcs(pVCpu, pVmcsInfo, fIsNestedGuest, &VCPU_2_VMXSTATE(pVCpu).Event, fStepping, &fIntrState);
6735	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
6736
6737	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6738	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectInterrupt);
6739	else
6740	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectXcpt);
6741	}
6742
6743	/*
6744	* Deliver any pending debug exceptions if the guest is single-stepping using EFLAGS.TF and
6745	* is an interrupt shadow (block-by-STI or block-by-MOV SS).
6746	*/
6747	if ( (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
6748	&& !fIsNestedGuest)
6749	{
6750	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
6751
6752	if (!VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
6753	{
6754	/*
6755	* Set or clear the BS bit depending on whether the trap flag is active or not. We need
6756	* to do both since we clear the BS bit from the VMCS while exiting to ring-3.
6757	*/
6758	Assert(!DBGFIsStepping(pVCpu));
6759	uint8_t const fTrapFlag = !!(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_TF);
6760	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, fTrapFlag << VMX_BF_VMCS_PENDING_DBG_XCPT_BS_SHIFT);
6761	AssertRC(rc);
6762	}
6763	else
6764	{
6765	/*
6766	* We must not deliver a debug exception when single-stepping over STI/Mov-SS in the
6767	* hypervisor debugger using EFLAGS.TF but rather clear interrupt inhibition. However,
6768	* we take care of this case in vmxHCExportSharedDebugState and also the case if
6769	* we use MTF, so just make sure it's called before executing guest-code.
6770	*/
6771	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_DR_MASK);
6772	}
6773	}
6774	/* else: for nested-guest currently handling while merging controls. */
6775
6776	/*
6777	* Finally, update the guest-interruptibility state.
6778	*
6779	* This is required for the real-on-v86 software interrupt injection, for
6780	* pending debug exceptions as well as updates to the guest state from ring-3 (IEM).
6781	*/
6782	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, fIntrState);
6783	AssertRC(rc);
6784
6785	/*
6786	* There's no need to clear the VM-entry interruption-information field here if we're not
6787	* injecting anything. VT-x clears the valid bit on every VM-exit.
6788	*
6789	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6790	*/
6791
6792	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
6793	return rcStrict;
6794	}
6795
6796
6797	#ifdef IN_RING0
6798	/**
6799	* Exports the guest state into the VMCS guest-state area.
6800	*
6801	* The will typically be done before VM-entry when the guest-CPU state and the
6802	* VMCS state may potentially be out of sync.
6803	*
6804	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
6805	* VM-entry controls.
6806	* Sets up the appropriate VMX non-root function to execute guest code based on
6807	* the guest CPU mode.
6808	*
6809	* @returns VBox strict status code.
6810	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
6811	* without unrestricted guest execution and the VMMDev is not presently
6812	* mapped (e.g. EFI32).
6813	*
6814	* @param pVCpu The cross context virtual CPU structure.
6815	* @param pVmxTransient The VMX-transient structure.
6816	*
6817	* @remarks No-long-jump zone!!!
6818	*/
6819	static VBOXSTRICTRC vmxHCExportGuestState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6820	{
6821	AssertPtr(pVCpu);
6822	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6823	LogFlowFunc(("pVCpu=%p\n", pVCpu));
6824
6825	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExportGuestState, x);
6826
6827	/*
6828	* Determine real-on-v86 mode.
6829	* Used when the guest is in real-mode and unrestricted guest execution is not used.
6830	*/
6831	PVMXVMCSINFOSHARED pVmcsInfoShared = pVmxTransient->pVmcsInfo->pShared;
6832	if ( pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fUnrestrictedGuest
6833	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
6834	pVmcsInfoShared->RealMode.fRealOnV86Active = false;
6835	else
6836	{
6837	Assert(!pVmxTransient->fIsNestedGuest);
6838	pVmcsInfoShared->RealMode.fRealOnV86Active = true;
6839	}
6840
6841	/*
6842	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
6843	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
6844	*/
6845	int rc = vmxHCExportGuestEntryExitCtls(pVCpu, pVmxTransient);
6846	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
6847
6848	rc = vmxHCExportGuestCR0(pVCpu, pVmxTransient);
6849	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
6850
6851	VBOXSTRICTRC rcStrict = vmxHCExportGuestCR3AndCR4(pVCpu, pVmxTransient);
6852	if (rcStrict == VINF_SUCCESS)
6853	{ /* likely */ }
6854	else
6855	{
6856	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
6857	return rcStrict;
6858	}
6859
6860	rc = vmxHCExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
6861	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
6862
6863	rc = vmxHCExportGuestMsrs(pVCpu, pVmxTransient);
6864	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
6865
6866	vmxHCExportGuestApicTpr(pVCpu, pVmxTransient);
6867	vmxHCExportGuestXcptIntercepts(pVCpu, pVmxTransient);
6868	vmxHCExportGuestRip(pVCpu);
6869	vmxHCExportGuestRsp(pVCpu);
6870	vmxHCExportGuestRflags(pVCpu, pVmxTransient);
6871
6872	rc = vmxHCExportGuestHwvirtState(pVCpu, pVmxTransient);
6873	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
6874
6875	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
6876	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
6877	\| HM_CHANGED_GUEST_CR2
6878	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
6879	\| HM_CHANGED_GUEST_X87
6880	\| HM_CHANGED_GUEST_SSE_AVX
6881	\| HM_CHANGED_GUEST_OTHER_XSAVE
6882	\| HM_CHANGED_GUEST_XCRx
6883	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
6884	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
6885	\| HM_CHANGED_GUEST_TSC_AUX
6886	\| HM_CHANGED_GUEST_OTHER_MSRS
6887	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
6888
6889	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExportGuestState, x);
6890	return rc;
6891	}
6892
6893
6894	/**
6895	* Exports the state shared between the host and guest into the VMCS.
6896	*
6897	* @param pVCpu The cross context virtual CPU structure.
6898	* @param pVmxTransient The VMX-transient structure.
6899	*
6900	* @remarks No-long-jump zone!!!
6901	*/
6902	static void vmxHCExportSharedState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6903	{
6904	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6905	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6906
6907	if (VCPU_2_VMXSTATE(pVCpu).fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
6908	{
6909	int rc = vmxHCExportSharedDebugState(pVCpu, pVmxTransient);
6910	AssertRC(rc);
6911	VCPU_2_VMXSTATE(pVCpu).fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
6912
6913	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
6914	if (VCPU_2_VMXSTATE(pVCpu).fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
6915	vmxHCExportGuestRflags(pVCpu, pVmxTransient);
6916	}
6917
6918	if (VCPU_2_VMXSTATE(pVCpu).fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
6919	{
6920	vmxHCLazyLoadGuestMsrs(pVCpu);
6921	VCPU_2_VMXSTATE(pVCpu).fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
6922	}
6923
6924	AssertMsg(!(VCPU_2_VMXSTATE(pVCpu).fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
6925	("fCtxChanged=%#RX64\n", VCPU_2_VMXSTATE(pVCpu).fCtxChanged));
6926	}
6927
6928
6929	/**
6930	* Worker for loading the guest-state bits in the inner VT-x execution loop.
6931	*
6932	* @returns Strict VBox status code (i.e. informational status codes too).
6933	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
6934	* without unrestricted guest execution and the VMMDev is not presently
6935	* mapped (e.g. EFI32).
6936	*
6937	* @param pVCpu The cross context virtual CPU structure.
6938	* @param pVmxTransient The VMX-transient structure.
6939	*
6940	* @remarks No-long-jump zone!!!
6941	*/
6942	static VBOXSTRICTRC vmxHCExportGuestStateOptimal(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6943	{
6944	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6945	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6946
6947	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
6948	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
6949	#endif
6950
6951	/*
6952	* For many VM-exits only RIP/RSP/RFLAGS (and HWVIRT state when executing a nested-guest)
6953	* changes. First try to export only these without going through all other changed-flag checks.
6954	*/
6955	VBOXSTRICTRC rcStrict;
6956	uint64_t const fCtxMask = HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE;
6957	uint64_t const fMinimalMask = HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT;
6958	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged);
6959
6960	/* If only RIP/RSP/RFLAGS/HWVIRT changed, export only those (quicker, happens more often).*/
6961	if ( (fCtxChanged & fMinimalMask)
6962	&& !(fCtxChanged & (fCtxMask & ~fMinimalMask)))
6963	{
6964	vmxHCExportGuestRip(pVCpu);
6965	vmxHCExportGuestRsp(pVCpu);
6966	vmxHCExportGuestRflags(pVCpu, pVmxTransient);
6967	rcStrict = vmxHCExportGuestHwvirtState(pVCpu, pVmxTransient);
6968	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExportMinimal);
6969	}
6970	/* If anything else also changed, go through the full export routine and export as required. */
6971	else if (fCtxChanged & fCtxMask)
6972	{
6973	rcStrict = vmxHCExportGuestState(pVCpu, pVmxTransient);
6974	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
6975	{ /* likely */}
6976	else
6977	{
6978	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
6979	VBOXSTRICTRC_VAL(rcStrict)));
6980	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6981	return rcStrict;
6982	}
6983	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExportFull);
6984	}
6985	/* Nothing changed, nothing to load here. */
6986	else
6987	rcStrict = VINF_SUCCESS;
6988
6989	#ifdef VBOX_STRICT
6990	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
6991	uint64_t const fCtxChangedCur = ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged);
6992	AssertMsg(!(fCtxChangedCur & fCtxMask), ("fCtxChangedCur=%#RX64\n", fCtxChangedCur));
6993	#endif
6994	return rcStrict;
6995	}
6996	#endif
6997
6998
6999	/**
7000	* Tries to determine what part of the guest-state VT-x has deemed as invalid
7001	* and update error record fields accordingly.
7002	*
7003	* @returns VMX_IGS_* error codes.
7004	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
7005	* wrong with the guest state.
7006	*
7007	* @param pVCpu The cross context virtual CPU structure.
7008	* @param pVmcsInfo The VMCS info. object.
7009	*
7010	* @remarks This function assumes our cache of the VMCS controls
7011	* are valid, i.e. vmxHCCheckCachedVmcsCtls() succeeded.
7012	*/
7013	static uint32_t vmxHCCheckGuestState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
7014	{
7015	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
7016	#define HMVMX_CHECK_BREAK(expr, err) do { \
7017	if (!(expr)) { uError = (err); break; } \
7018	} while (0)
7019
7020	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7021	uint32_t uError = VMX_IGS_ERROR;
7022	uint32_t u32IntrState = 0;
7023	#ifdef IN_RING0
7024	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7025	bool const fUnrestrictedGuest = VM_IS_VMX_UNRESTRICTED_GUEST(pVM);
7026	#else
7027	bool const fUnrestrictedGuest = true;
7028	#endif
7029	do
7030	{
7031	int rc;
7032
7033	/*
7034	* Guest-interruptibility state.
7035	*
7036	* Read this first so that any check that fails prior to those that actually
7037	* require the guest-interruptibility state would still reflect the correct
7038	* VMCS value and avoids causing further confusion.
7039	*/
7040	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
7041	AssertRC(rc);
7042
7043	uint32_t u32Val;
7044	uint64_t u64Val;
7045
7046	/*
7047	* CR0.
7048	*/
7049	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
7050	uint64_t fSetCr0 = (g_HmMsrs.u.vmx.u64Cr0Fixed0 & g_HmMsrs.u.vmx.u64Cr0Fixed1);
7051	uint64_t const fZapCr0 = (g_HmMsrs.u.vmx.u64Cr0Fixed0 \| g_HmMsrs.u.vmx.u64Cr0Fixed1);
7052	/* Exceptions for unrestricted guest execution for CR0 fixed bits (PE, PG).
7053	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
7054	if (fUnrestrictedGuest)
7055	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
7056
7057	uint64_t u64GuestCr0;
7058	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0, &u64GuestCr0);
7059	AssertRC(rc);
7060	HMVMX_CHECK_BREAK((u64GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
7061	HMVMX_CHECK_BREAK(!(u64GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
7062	if ( !fUnrestrictedGuest
7063	&& (u64GuestCr0 & X86_CR0_PG)
7064	&& !(u64GuestCr0 & X86_CR0_PE))
7065	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
7066
7067	/*
7068	* CR4.
7069	*/
7070	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
7071	uint64_t const fSetCr4 = (g_HmMsrs.u.vmx.u64Cr4Fixed0 & g_HmMsrs.u.vmx.u64Cr4Fixed1);
7072	uint64_t const fZapCr4 = (g_HmMsrs.u.vmx.u64Cr4Fixed0 \| g_HmMsrs.u.vmx.u64Cr4Fixed1);
7073
7074	uint64_t u64GuestCr4;
7075	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR4, &u64GuestCr4);
7076	AssertRC(rc);
7077	HMVMX_CHECK_BREAK((u64GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
7078	HMVMX_CHECK_BREAK(!(u64GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
7079
7080	/*
7081	* IA32_DEBUGCTL MSR.
7082	*/
7083	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
7084	AssertRC(rc);
7085	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
7086	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
7087	{
7088	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
7089	}
7090	uint64_t u64DebugCtlMsr = u64Val;
7091
7092	#ifdef VBOX_STRICT
7093	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY, &u32Val);
7094	AssertRC(rc);
7095	Assert(u32Val == pVmcsInfo->u32EntryCtls);
7096	#endif
7097	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
7098
7099	/*
7100	* RIP and RFLAGS.
7101	*/
7102	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RIP, &u64Val);
7103	AssertRC(rc);
7104	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
7105	if ( !fLongModeGuest
7106	\|\| !pCtx->cs.Attr.n.u1Long)
7107	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
7108	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
7109	* must be identical if the "IA-32e mode guest" VM-entry
7110	* control is 1 and CS.L is 1. No check applies if the
7111	* CPU supports 64 linear-address bits. */
7112
7113	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
7114	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RFLAGS, &u64Val);
7115	AssertRC(rc);
7116	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
7117	VMX_IGS_RFLAGS_RESERVED);
7118	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
7119	uint32_t const u32Eflags = u64Val;
7120
7121	if ( fLongModeGuest
7122	\|\| ( fUnrestrictedGuest
7123	&& !(u64GuestCr0 & X86_CR0_PE)))
7124	{
7125	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
7126	}
7127
7128	uint32_t u32EntryInfo;
7129	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7130	AssertRC(rc);
7131	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
7132	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
7133
7134	/*
7135	* 64-bit checks.
7136	*/
7137	if (fLongModeGuest)
7138	{
7139	HMVMX_CHECK_BREAK(u64GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
7140	HMVMX_CHECK_BREAK(u64GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
7141	}
7142
7143	if ( !fLongModeGuest
7144	&& (u64GuestCr4 & X86_CR4_PCIDE))
7145	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
7146
7147	/** @todo CR3 field must be such that bits 63:52 and bits in the range
7148	* 51:32 beyond the processor's physical-address width are 0. */
7149
7150	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
7151	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
7152	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
7153
7154	#ifdef IN_RING0
7155	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
7156	AssertRC(rc);
7157	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
7158
7159	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
7160	AssertRC(rc);
7161	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
7162	#endif
7163
7164	/*
7165	* PERF_GLOBAL MSR.
7166	*/
7167	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
7168	{
7169	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
7170	AssertRC(rc);
7171	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
7172	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
7173	}
7174
7175	/*
7176	* PAT MSR.
7177	*/
7178	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
7179	{
7180	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
7181	AssertRC(rc);
7182	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
7183	for (unsigned i = 0; i < 8; i++)
7184	{
7185	uint8_t u8Val = (u64Val & 0xff);
7186	if ( u8Val != 0 /* UC */
7187	&& u8Val != 1 /* WC */
7188	&& u8Val != 4 /* WT */
7189	&& u8Val != 5 /* WP */
7190	&& u8Val != 6 /* WB */
7191	&& u8Val != 7 /* UC- */)
7192	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
7193	u64Val >>= 8;
7194	}
7195	}
7196
7197	/*
7198	* EFER MSR.
7199	*/
7200	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
7201	{
7202	Assert(g_fHmVmxSupportsVmcsEfer);
7203	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
7204	AssertRC(rc);
7205	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
7206	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
7207	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
7208	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
7209	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
7210	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
7211	* iemVmxVmentryCheckGuestState(). */
7212	HMVMX_CHECK_BREAK( fUnrestrictedGuest
7213	\|\| !(u64GuestCr0 & X86_CR0_PG)
7214	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
7215	VMX_IGS_EFER_LMA_LME_MISMATCH);
7216	}
7217
7218	/*
7219	* Segment registers.
7220	*/
7221	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
7222	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
7223	if (!(u32Eflags & X86_EFL_VM))
7224	{
7225	/* CS */
7226	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
7227	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
7228	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
7229	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
7230	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
7231	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
7232	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
7233	/* CS cannot be loaded with NULL in protected mode. */
7234	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
7235	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
7236	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
7237	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
7238	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
7239	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
7240	else if (fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
7241	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
7242	else
7243	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
7244
7245	/* SS */
7246	HMVMX_CHECK_BREAK( fUnrestrictedGuest
7247	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
7248	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
7249	if ( !(pCtx->cr0 & X86_CR0_PE)
7250	\|\| pCtx->cs.Attr.n.u4Type == 3)
7251	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
7252
7253	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
7254	{
7255	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
7256	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
7257	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
7258	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
7259	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
7260	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
7261	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
7262	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
7263	}
7264
7265	/* DS, ES, FS, GS - only check for usable selectors, see vmxHCExportGuestSReg(). */
7266	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
7267	{
7268	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
7269	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
7270	HMVMX_CHECK_BREAK( fUnrestrictedGuest
7271	\|\| pCtx->ds.Attr.n.u4Type > 11
7272	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
7273	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
7274	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
7275	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
7276	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
7277	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
7278	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
7279	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
7280	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
7281	}
7282	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
7283	{
7284	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
7285	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
7286	HMVMX_CHECK_BREAK( fUnrestrictedGuest
7287	\|\| pCtx->es.Attr.n.u4Type > 11
7288	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
7289	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
7290	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
7291	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
7292	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
7293	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
7294	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
7295	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
7296	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
7297	}
7298	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
7299	{
7300	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
7301	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
7302	HMVMX_CHECK_BREAK( fUnrestrictedGuest
7303	\|\| pCtx->fs.Attr.n.u4Type > 11
7304	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
7305	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
7306	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
7307	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
7308	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
7309	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
7310	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
7311	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
7312	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
7313	}
7314	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
7315	{
7316	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
7317	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
7318	HMVMX_CHECK_BREAK( fUnrestrictedGuest
7319	\|\| pCtx->gs.Attr.n.u4Type > 11
7320	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
7321	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
7322	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
7323	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
7324	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
7325	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
7326	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
7327	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
7328	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
7329	}
7330	/* 64-bit capable CPUs. */
7331	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
7332	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
7333	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
7334	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
7335	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
7336	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
7337	VMX_IGS_LONGMODE_SS_BASE_INVALID);
7338	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
7339	VMX_IGS_LONGMODE_DS_BASE_INVALID);
7340	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
7341	VMX_IGS_LONGMODE_ES_BASE_INVALID);
7342	}
7343	else
7344	{
7345	/* V86 mode checks. */
7346	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
7347	if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
7348	{
7349	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
7350	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
7351	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
7352	}
7353	else
7354	{
7355	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
7356	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
7357	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
7358	}
7359
7360	/* CS */
7361	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
7362	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
7363	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
7364	/* SS */
7365	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
7366	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
7367	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
7368	/* DS */
7369	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
7370	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
7371	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
7372	/* ES */
7373	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
7374	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
7375	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
7376	/* FS */
7377	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
7378	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
7379	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
7380	/* GS */
7381	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
7382	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
7383	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
7384	/* 64-bit capable CPUs. */
7385	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
7386	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
7387	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
7388	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
7389	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
7390	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
7391	VMX_IGS_LONGMODE_SS_BASE_INVALID);
7392	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
7393	VMX_IGS_LONGMODE_DS_BASE_INVALID);
7394	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
7395	VMX_IGS_LONGMODE_ES_BASE_INVALID);
7396	}
7397
7398	/*
7399	* TR.
7400	*/
7401	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
7402	/* 64-bit capable CPUs. */
7403	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
7404	if (fLongModeGuest)
7405	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
7406	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
7407	else
7408	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
7409	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
7410	VMX_IGS_TR_ATTR_TYPE_INVALID);
7411	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
7412	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
7413	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
7414	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
7415	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
7416	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
7417	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
7418	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
7419
7420	/*
7421	* GDTR and IDTR (64-bit capable checks).
7422	*/
7423	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
7424	AssertRC(rc);
7425	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
7426
7427	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
7428	AssertRC(rc);
7429	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
7430
7431	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
7432	AssertRC(rc);
7433	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
7434
7435	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
7436	AssertRC(rc);
7437	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
7438
7439	/*
7440	* Guest Non-Register State.
7441	*/
7442	/* Activity State. */
7443	uint32_t u32ActivityState;
7444	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
7445	AssertRC(rc);
7446	HMVMX_CHECK_BREAK( !u32ActivityState
7447	\|\| (u32ActivityState & RT_BF_GET(g_HmMsrs.u.vmx.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
7448	VMX_IGS_ACTIVITY_STATE_INVALID);
7449	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
7450	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
7451
7452	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
7453	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
7454	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
7455
7456	/** @todo Activity state and injecting interrupts. Left as a todo since we
7457	* currently don't use activity states but ACTIVE. */
7458
7459	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
7460	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
7461
7462	/* Guest interruptibility-state. */
7463	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
7464	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
7465	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
7466	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
7467	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
7468	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
7469	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
7470	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
7471	{
7472	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
7473	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
7474	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
7475	}
7476	else if (VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
7477	{
7478	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
7479	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
7480	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
7481	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
7482	}
7483	/** @todo Assumes the processor is not in SMM. */
7484	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
7485	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
7486	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
7487	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
7488	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
7489	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
7490	&& VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
7491	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI), VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
7492
7493	/* Pending debug exceptions. */
7494	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
7495	AssertRC(rc);
7496	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
7497	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
7498	u32Val = u64Val; /* For pending debug exceptions checks below. */
7499
7500	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
7501	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
7502	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
7503	{
7504	if ( (u32Eflags & X86_EFL_TF)
7505	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
7506	{
7507	/* Bit 14 is PendingDebug.BS. */
7508	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
7509	}
7510	if ( !(u32Eflags & X86_EFL_TF)
7511	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
7512	{
7513	/* Bit 14 is PendingDebug.BS. */
7514	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
7515	}
7516	}
7517
7518	#ifdef IN_RING0
7519	/* VMCS link pointer. */
7520	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
7521	AssertRC(rc);
7522	if (u64Val != UINT64_C(0xffffffffffffffff))
7523	{
7524	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
7525	/** @todo Bits beyond the processor's physical-address width MBZ. */
7526	/** @todo SMM checks. */
7527	Assert(pVmcsInfo->HCPhysShadowVmcs == u64Val);
7528	Assert(pVmcsInfo->pvShadowVmcs);
7529	VMXVMCSREVID VmcsRevId;
7530	VmcsRevId.u = (uint32_t )pVmcsInfo->pvShadowVmcs;
7531	HMVMX_CHECK_BREAK(VmcsRevId.n.u31RevisionId == RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_ID),
7532	VMX_IGS_VMCS_LINK_PTR_SHADOW_VMCS_ID_INVALID);
7533	HMVMX_CHECK_BREAK(VmcsRevId.n.fIsShadowVmcs == (uint32_t)!!(pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING),
7534	VMX_IGS_VMCS_LINK_PTR_NOT_SHADOW);
7535	}
7536
7537	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
7538	* not using nested paging? */
7539	if ( VM_IS_VMX_NESTED_PAGING(pVM)
7540	&& !fLongModeGuest
7541	&& CPUMIsGuestInPAEModeEx(pCtx))
7542	{
7543	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
7544	AssertRC(rc);
7545	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
7546
7547	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
7548	AssertRC(rc);
7549	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
7550
7551	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
7552	AssertRC(rc);
7553	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
7554
7555	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
7556	AssertRC(rc);
7557	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
7558	}
7559	#endif
7560
7561	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
7562	if (uError == VMX_IGS_ERROR)
7563	uError = VMX_IGS_REASON_NOT_FOUND;
7564	} while (0);
7565
7566	VCPU_2_VMXSTATE(pVCpu).u32HMError = uError;
7567	VCPU_2_VMXSTATE(pVCpu).vmx.LastError.u32GuestIntrState = u32IntrState;
7568	return uError;
7569
7570	#undef HMVMX_ERROR_BREAK
7571	#undef HMVMX_CHECK_BREAK
7572	}
7573
7574
7575	#ifdef IN_RING0
7576	/**
7577	* Map the APIC-access page for virtualizing APIC accesses.
7578	*
7579	* This can cause a longjumps to R3 due to the acquisition of the PGM lock. Hence,
7580	* this not done as part of exporting guest state, see @bugref{8721}.
7581	*
7582	* @returns VBox status code.
7583	* @param pVCpu The cross context virtual CPU structure.
7584	*/
7585	static int vmxHCMapHCApicAccessPage(PVMCPUCC pVCpu)
7586	{
7587	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7588	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
7589
7590	Assert(PDMHasApic(pVM));
7591	Assert(u64MsrApicBase);
7592
7593	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
7594	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
7595
7596	/* Unalias the existing mapping. */
7597	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7598	AssertRCReturn(rc, rc);
7599
7600	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
7601	Assert(pVM->hmr0.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
7602	rc = IOMR0MmioMapMmioHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hmr0.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
7603	AssertRCReturn(rc, rc);
7604
7605	/* Update the per-VCPU cache of the APIC base MSR. */
7606	VCPU_2_VMXSTATE(pVCpu).vmx.u64GstMsrApicBase = u64MsrApicBase;
7607	return VINF_SUCCESS;
7608	}
7609
7610
7611	/**
7612	* Worker function passed to RTMpOnSpecific() that is to be called on the target
7613	* CPU.
7614	*
7615	* @param idCpu The ID for the CPU the function is called on.
7616	* @param pvUser1 Null, not used.
7617	* @param pvUser2 Null, not used.
7618	*/
7619	static DECLCALLBACK(void) hmR0DispatchHostNmi(RTCPUID idCpu, void pvUser1, void pvUser2)
7620	{
7621	RT_NOREF3(idCpu, pvUser1, pvUser2);
7622	VMXDispatchHostNmi();
7623	}
7624
7625
7626	/**
7627	* Dispatching an NMI on the host CPU that received it.
7628	*
7629	* @returns VBox status code.
7630	* @param pVCpu The cross context virtual CPU structure.
7631	* @param pVmcsInfo The VMCS info. object corresponding to the VMCS that was
7632	* executing when receiving the host NMI in VMX non-root
7633	* operation.
7634	*/
7635	static int vmxHCExitHostNmi(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
7636	{
7637	RTCPUID const idCpu = pVmcsInfo->idHostCpuExec;
7638	Assert(idCpu != NIL_RTCPUID);
7639
7640	/*
7641	* We don't want to delay dispatching the NMI any more than we have to. However,
7642	* we have already chosen -not- to dispatch NMIs when interrupts were still disabled
7643	* after executing guest or nested-guest code for the following reasons:
7644	*
7645	* - We would need to perform VMREADs with interrupts disabled and is orders of
7646	* magnitude worse when we run as a nested hypervisor without VMCS shadowing
7647	* supported by the host hypervisor.
7648	*
7649	* - It affects the common VM-exit scenario and keeps interrupts disabled for a
7650	* longer period of time just for handling an edge case like host NMIs which do
7651	* not occur nearly as frequently as other VM-exits.
7652	*
7653	* Let's cover the most likely scenario first. Check if we are on the target CPU
7654	* and dispatch the NMI right away. This should be much faster than calling into
7655	* RTMpOnSpecific() machinery.
7656	*/
7657	bool fDispatched = false;
7658	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7659	if (idCpu == RTMpCpuId())
7660	{
7661	VMXDispatchHostNmi();
7662	fDispatched = true;
7663	}
7664	ASMSetFlags(fEFlags);
7665	if (fDispatched)
7666	{
7667	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitHostNmiInGC);
7668	return VINF_SUCCESS;
7669	}
7670
7671	/*
7672	* RTMpOnSpecific() waits until the worker function has run on the target CPU. So
7673	* there should be no race or recursion even if we are unlucky enough to be preempted
7674	* (to the target CPU) without dispatching the host NMI above.
7675	*/
7676	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitHostNmiInGCIpi);
7677	return RTMpOnSpecific(idCpu, &hmR0DispatchHostNmi, NULL /* pvUser1 /, NULL / pvUser2 */);
7678	}
7679
7680
7681	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7682	/**
7683	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
7684	* nested-guest using hardware-assisted VMX.
7685	*
7686	* @param pVCpu The cross context virtual CPU structure.
7687	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
7688	* @param pVmcsInfoGst The guest VMCS info. object.
7689	*/
7690	static void vmxHCMergeMsrBitmapNested(PCVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
7691	{
7692	uint32_t const cbMsrBitmap = X86_PAGE_4K_SIZE;
7693	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
7694	Assert(pu64MsrBitmap);
7695
7696	/*
7697	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
7698	* MSR that is intercepted by the guest is also intercepted while executing the
7699	* nested-guest using hardware-assisted VMX.
7700	*
7701	* Note! If the nested-guest is not using an MSR bitmap, every MSR must cause a
7702	* nested-guest VM-exit even if the outer guest is not intercepting some
7703	* MSRs. We cannot assume the caller has initialized the nested-guest
7704	* MSR bitmap in this case.
7705	*
7706	* The nested hypervisor may also switch whether it uses MSR bitmaps for
7707	* each of its VM-entry, hence initializing it once per-VM while setting
7708	* up the nested-guest VMCS is not sufficient.
7709	*/
7710	PCVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
7711	if (pVmcsNstGst->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
7712	{
7713	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )&pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap[0];
7714	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
7715	Assert(pu64MsrBitmapNstGst);
7716	Assert(pu64MsrBitmapGst);
7717
7718	/** @todo Detect and use EVEX.POR? */
7719	uint32_t const cFrags = cbMsrBitmap / sizeof(uint64_t);
7720	for (uint32_t i = 0; i < cFrags; i++)
7721	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
7722	}
7723	else
7724	ASMMemFill32(pu64MsrBitmap, cbMsrBitmap, UINT32_C(0xffffffff));
7725	}
7726
7727
7728	/**
7729	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
7730	* hardware-assisted VMX execution of the nested-guest.
7731	*
7732	* For a guest, we don't modify these controls once we set up the VMCS and hence
7733	* this function is never called.
7734	*
7735	* For nested-guests since the nested hypervisor provides these controls on every
7736	* nested-guest VM-entry and could potentially change them everytime we need to
7737	* merge them before every nested-guest VM-entry.
7738	*
7739	* @returns VBox status code.
7740	* @param pVCpu The cross context virtual CPU structure.
7741	*/
7742	static int vmxHCMergeVmcsNested(PVMCPUCC pVCpu)
7743	{
7744	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
7745	PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo;
7746	PCVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
7747
7748	/*
7749	* Merge the controls with the requirements of the guest VMCS.
7750	*
7751	* We do not need to validate the nested-guest VMX features specified in the nested-guest
7752	* VMCS with the features supported by the physical CPU as it's already done by the
7753	* VMLAUNCH/VMRESUME instruction emulation.
7754	*
7755	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the guest are
7756	* derived from the VMX features supported by the physical CPU.
7757	*/
7758
7759	/* Pin-based VM-execution controls. */
7760	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
7761
7762	/* Processor-based VM-execution controls. */
7763	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
7764	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
7765	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
7766	\| VMX_PROC_CTLS_MOV_DR_EXIT
7767	\| VMX_PROC_CTLS_USE_TPR_SHADOW
7768	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
7769
7770	/* Secondary processor-based VM-execution controls. */
7771	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
7772	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
7773	\| VMX_PROC_CTLS2_INVPCID
7774	\| VMX_PROC_CTLS2_VMCS_SHADOWING
7775	\| VMX_PROC_CTLS2_RDTSCP
7776	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
7777	\| VMX_PROC_CTLS2_APIC_REG_VIRT
7778	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
7779	\| VMX_PROC_CTLS2_VMFUNC));
7780
7781	/*
7782	* VM-entry controls:
7783	* These controls contains state that depends on the nested-guest state (primarily
7784	* EFER MSR) and is thus not constant between VMLAUNCH/VMRESUME and the nested-guest
7785	* VM-exit. Although the nested hypervisor cannot change it, we need to in order to
7786	* properly continue executing the nested-guest if the EFER MSR changes but does not
7787	* cause a nested-guest VM-exits.
7788	*
7789	* VM-exit controls:
7790	* These controls specify the host state on return. We cannot use the controls from
7791	* the nested hypervisor state as is as it would contain the guest state rather than
7792	* the host state. Since the host state is subject to change (e.g. preemption, trips
7793	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
7794	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
7795	*
7796	* VM-entry MSR-load:
7797	* The guest MSRs from the VM-entry MSR-load area are already loaded into the guest-CPU
7798	* context by the VMLAUNCH/VMRESUME instruction emulation.
7799	*
7800	* VM-exit MSR-store:
7801	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU context
7802	* back into the VM-exit MSR-store area.
7803	*
7804	* VM-exit MSR-load areas:
7805	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence, we
7806	* can entirely ignore what the nested hypervisor wants to load here.
7807	*/
7808
7809	/*
7810	* Exception bitmap.
7811	*
7812	* We could remove #UD from the guest bitmap and merge it with the nested-guest bitmap
7813	* here (and avoid doing anything while exporting nested-guest state), but to keep the
7814	* code more flexible if intercepting exceptions become more dynamic in the future we do
7815	* it as part of exporting the nested-guest state.
7816	*/
7817	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
7818
7819	/*
7820	* CR0/CR4 guest/host mask.
7821	*
7822	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest must
7823	* cause VM-exits, so we need to merge them here.
7824	*/
7825	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
7826	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
7827
7828	/*
7829	* Page-fault error-code mask and match.
7830	*
7831	* Although we require unrestricted guest execution (and thereby nested-paging) for
7832	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
7833	* normally intercept #PFs, it might intercept them for debugging purposes.
7834	*
7835	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF filters.
7836	* If the outer guest is intercepting #PFs, we must intercept all #PFs.
7837	*/
7838	uint32_t u32XcptPFMask;
7839	uint32_t u32XcptPFMatch;
7840	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
7841	{
7842	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
7843	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
7844	}
7845	else
7846	{
7847	u32XcptPFMask = 0;
7848	u32XcptPFMatch = 0;
7849	}
7850
7851	/*
7852	* Pause-Loop exiting.
7853	*/
7854	/** @todo r=bird: given that both pVM->hm.s.vmx.cPleGapTicks and
7855	* pVM->hm.s.vmx.cPleWindowTicks defaults to zero, I cannot see how
7856	* this will work... */
7857	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
7858	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
7859
7860	/*
7861	* Pending debug exceptions.
7862	* Currently just copy whatever the nested-guest provides us.
7863	*/
7864	uint64_t const uPendingDbgXcpts = pVmcsNstGst->u64GuestPendingDbgXcpts.u;
7865
7866	/*
7867	* I/O Bitmap.
7868	*
7869	* We do not use the I/O bitmap that may be provided by the nested hypervisor as we always
7870	* intercept all I/O port accesses.
7871	*/
7872	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
7873	Assert(!(u32ProcCtls & VMX_PROC_CTLS_USE_IO_BITMAPS));
7874
7875	/*
7876	* VMCS shadowing.
7877	*
7878	* We do not yet expose VMCS shadowing to the guest and thus VMCS shadowing should not be
7879	* enabled while executing the nested-guest.
7880	*/
7881	Assert(!(u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING));
7882
7883	/*
7884	* APIC-access page.
7885	*/
7886	RTHCPHYS HCPhysApicAccess;
7887	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
7888	{
7889	Assert(g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS);
7890	RTGCPHYS const GCPhysApicAccess = pVmcsNstGst->u64AddrApicAccess.u;
7891
7892	/** @todo NSTVMX: This is not really correct but currently is required to make
7893	* things work. We need to re-enable the page handler when we fallback to
7894	* IEM execution of the nested-guest! */
7895	PGMHandlerPhysicalPageTempOff(pVM, GCPhysApicAccess, GCPhysApicAccess);
7896
7897	void *pvPage;
7898	PGMPAGEMAPLOCK PgLockApicAccess;
7899	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysApicAccess, &pvPage, &PgLockApicAccess);
7900	if (RT_SUCCESS(rc))
7901	{
7902	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysApicAccess, &HCPhysApicAccess);
7903	AssertMsgRCReturn(rc, ("Failed to get host-physical address for APIC-access page at %#RGp\n", GCPhysApicAccess), rc);
7904
7905	/** @todo Handle proper releasing of page-mapping lock later. */
7906	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &PgLockApicAccess);
7907	}
7908	else
7909	return rc;
7910	}
7911	else
7912	HCPhysApicAccess = 0;
7913
7914	/*
7915	* Virtual-APIC page and TPR threshold.
7916	*/
7917	RTHCPHYS HCPhysVirtApic;
7918	uint32_t u32TprThreshold;
7919	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
7920	{
7921	Assert(g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW);
7922	RTGCPHYS const GCPhysVirtApic = pVmcsNstGst->u64AddrVirtApic.u;
7923
7924	void *pvPage;
7925	PGMPAGEMAPLOCK PgLockVirtApic;
7926	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysVirtApic, &pvPage, &PgLockVirtApic);
7927	if (RT_SUCCESS(rc))
7928	{
7929	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysVirtApic, &HCPhysVirtApic);
7930	AssertMsgRCReturn(rc, ("Failed to get host-physical address for virtual-APIC page at %#RGp\n", GCPhysVirtApic), rc);
7931
7932	/** @todo Handle proper releasing of page-mapping lock later. */
7933	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &PgLockVirtApic);
7934	}
7935	else
7936	return rc;
7937
7938	u32TprThreshold = pVmcsNstGst->u32TprThreshold;
7939	}
7940	else
7941	{
7942	HCPhysVirtApic = 0;
7943	u32TprThreshold = 0;
7944
7945	/*
7946	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
7947	* used by the nested hypervisor. Preventing MMIO accesses to the physical APIC will
7948	* be taken care of by EPT/shadow paging.
7949	*/
7950	if (pVM->hmr0.s.fAllow64BitGuests)
7951	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
7952	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
7953	}
7954
7955	/*
7956	* Validate basic assumptions.
7957	*/
7958	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hmr0.s.vmx.VmcsInfoNstGst;
7959	Assert(VM_IS_VMX_UNRESTRICTED_GUEST(pVM));
7960	Assert(g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
7961	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
7962
7963	/*
7964	* Commit it to the nested-guest VMCS.
7965	*/
7966	int rc = VINF_SUCCESS;
7967	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
7968	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
7969	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
7970	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
7971	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
7972	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
7973	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
7974	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
7975	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
7976	rc \|= VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
7977	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
7978	rc \|= VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
7979	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
7980	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
7981	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
7982	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
7983	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
7984	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
7985	{
7986	Assert(g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
7987	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
7988	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
7989	}
7990	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
7991	{
7992	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
7993	rc \|= VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
7994	}
7995	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
7996	rc \|= VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
7997	rc \|= VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, uPendingDbgXcpts);
7998	AssertRC(rc);
7999
8000	/*
8001	* Update the nested-guest VMCS cache.
8002	*/
8003	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
8004	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
8005	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
8006	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
8007	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
8008	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
8009	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
8010	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
8011	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
8012
8013	/*
8014	* We need to flush the TLB if we are switching the APIC-access page address.
8015	* See Intel spec. 28.3.3.4 "Guidelines for Use of the INVEPT Instruction".
8016	*/
8017	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
8018	VCPU_2_VMXSTATE(pVCpu).vmx.fSwitchedNstGstFlushTlb = true;
8019
8020	/*
8021	* MSR bitmap.
8022	*
8023	* The MSR bitmap address has already been initialized while setting up the nested-guest
8024	* VMCS, here we need to merge the MSR bitmaps.
8025	*/
8026	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
8027	vmxHCMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
8028
8029	return VINF_SUCCESS;
8030	}
8031	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
8032
8033
8034	/**
8035	* Runs the guest code using hardware-assisted VMX the normal way.
8036	*
8037	* @returns VBox status code.
8038	* @param pVCpu The cross context virtual CPU structure.
8039	* @param pcLoops Pointer to the number of executed loops.
8040	*/
8041	static VBOXSTRICTRC vmxHCRunGuestCodeNormal(PVMCPUCC pVCpu, uint32_t *pcLoops)
8042	{
8043	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
8044	Assert(pcLoops);
8045	Assert(*pcLoops <= cMaxResumeLoops);
8046	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8047
8048	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8049	/*
8050	* Switch to the guest VMCS as we may have transitioned from executing the nested-guest
8051	* without leaving ring-0. Otherwise, if we came from ring-3 we would have loaded the
8052	* guest VMCS while entering the VMX ring-0 session.
8053	*/
8054	if (pVCpu->hmr0.s.vmx.fSwitchedToNstGstVmcs)
8055	{
8056	int rc = vmxHCSwitchToGstOrNstGstVmcs(pVCpu, false /* fSwitchToNstGstVmcs */);
8057	if (RT_SUCCESS(rc))
8058	{ /* likely */ }
8059	else
8060	{
8061	LogRelFunc(("Failed to switch to the guest VMCS. rc=%Rrc\n", rc));
8062	return rc;
8063	}
8064	}
8065	#endif
8066
8067	VMXTRANSIENT VmxTransient;
8068	RT_ZERO(VmxTransient);
8069	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8070
8071	/* Paranoia. */
8072	Assert(VmxTransient.pVmcsInfo == &pVCpu->hmr0.s.vmx.VmcsInfo);
8073
8074	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8075	for (;;)
8076	{
8077	Assert(!HMR0SuspendPending());
8078	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8079	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatEntry, x);
8080
8081	/*
8082	* Preparatory work for running nested-guest code, this may force us to
8083	* return to ring-3.
8084	*
8085	* Warning! This bugger disables interrupts on VINF_SUCCESS!
8086	*/
8087	rcStrict = vmxHCPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
8088	if (rcStrict != VINF_SUCCESS)
8089	break;
8090
8091	/* Interrupts are disabled at this point! */
8092	vmxHCPreRunGuestCommitted(pVCpu, &VmxTransient);
8093	int rcRun = vmxHCRunGuest(pVCpu, &VmxTransient);
8094	vmxHCPostRunGuest(pVCpu, &VmxTransient, rcRun);
8095	/* Interrupts are re-enabled at this point! */
8096
8097	/*
8098	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
8099	*/
8100	if (RT_SUCCESS(rcRun))
8101	{ /* very likely */ }
8102	else
8103	{
8104	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatPreExit, x);
8105	vmxHCReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
8106	return rcRun;
8107	}
8108
8109	/*
8110	* Profile the VM-exit.
8111	*/
8112	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8113	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitAll);
8114	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).aStatExitReason[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8115	STAM_PROFILE_ADV_STOP_START(&VCPU_2_VMXSTATS(pVCpu).StatPreExit, &VCPU_2_VMXSTATS(pVCpu).StatExitHandling, x);
8116	HMVMX_START_EXIT_DISPATCH_PROF();
8117
8118	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
8119
8120	/*
8121	* Handle the VM-exit.
8122	*/
8123	#ifdef HMVMX_USE_FUNCTION_TABLE
8124	rcStrict = g_aVMExitHandlers[VmxTransient.uExitReason].pfn(pVCpu, &VmxTransient);
8125	#else
8126	rcStrict = vmxHCHandleExit(pVCpu, &VmxTransient);
8127	#endif
8128	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitHandling, x);
8129	if (rcStrict == VINF_SUCCESS)
8130	{
8131	if (++(*pcLoops) <= cMaxResumeLoops)
8132	continue;
8133	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchMaxResumeLoops);
8134	rcStrict = VINF_EM_RAW_INTERRUPT;
8135	}
8136	break;
8137	}
8138
8139	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatEntry, x);
8140	return rcStrict;
8141	}
8142
8143
8144	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8145	/**
8146	* Runs the nested-guest code using hardware-assisted VMX.
8147	*
8148	* @returns VBox status code.
8149	* @param pVCpu The cross context virtual CPU structure.
8150	* @param pcLoops Pointer to the number of executed loops.
8151	*
8152	* @sa vmxHCRunGuestCodeNormal.
8153	*/
8154	static VBOXSTRICTRC vmxHCRunGuestCodeNested(PVMCPUCC pVCpu, uint32_t *pcLoops)
8155	{
8156	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
8157	Assert(pcLoops);
8158	Assert(*pcLoops <= cMaxResumeLoops);
8159	Assert(CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8160
8161	/*
8162	* Switch to the nested-guest VMCS as we may have transitioned from executing the
8163	* guest without leaving ring-0. Otherwise, if we came from ring-3 we would have
8164	* loaded the nested-guest VMCS while entering the VMX ring-0 session.
8165	*/
8166	if (!pVCpu->hmr0.s.vmx.fSwitchedToNstGstVmcs)
8167	{
8168	int rc = vmxHCSwitchToGstOrNstGstVmcs(pVCpu, true /* fSwitchToNstGstVmcs */);
8169	if (RT_SUCCESS(rc))
8170	{ /* likely */ }
8171	else
8172	{
8173	LogRelFunc(("Failed to switch to the nested-guest VMCS. rc=%Rrc\n", rc));
8174	return rc;
8175	}
8176	}
8177
8178	VMXTRANSIENT VmxTransient;
8179	RT_ZERO(VmxTransient);
8180	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8181	VmxTransient.fIsNestedGuest = true;
8182
8183	/* Paranoia. */
8184	Assert(VmxTransient.pVmcsInfo == &pVCpu->hmr0.s.vmx.VmcsInfoNstGst);
8185
8186	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8187	for (;;)
8188	{
8189	Assert(!HMR0SuspendPending());
8190	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8191	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatEntry, x);
8192
8193	/*
8194	* Preparatory work for running guest code, this may force us to
8195	* return to ring-3.
8196	*
8197	* Warning! This bugger disables interrupts on VINF_SUCCESS!
8198	*/
8199	rcStrict = vmxHCPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
8200	if (rcStrict != VINF_SUCCESS)
8201	break;
8202
8203	/* Interrupts are disabled at this point! */
8204	vmxHCPreRunGuestCommitted(pVCpu, &VmxTransient);
8205	int rcRun = vmxHCRunGuest(pVCpu, &VmxTransient);
8206	vmxHCPostRunGuest(pVCpu, &VmxTransient, rcRun);
8207	/* Interrupts are re-enabled at this point! */
8208
8209	/*
8210	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
8211	*/
8212	if (RT_SUCCESS(rcRun))
8213	{ /* very likely */ }
8214	else
8215	{
8216	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatPreExit, x);
8217	vmxHCReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
8218	return rcRun;
8219	}
8220
8221	/*
8222	* Profile the VM-exit.
8223	*/
8224	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8225	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitAll);
8226	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatNestedExitAll);
8227	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).aStatNestedExitReason[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8228	STAM_PROFILE_ADV_STOP_START(&VCPU_2_VMXSTATS(pVCpu).StatPreExit, &VCPU_2_VMXSTATS(pVCpu).StatExitHandling, x);
8229	HMVMX_START_EXIT_DISPATCH_PROF();
8230
8231	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
8232
8233	/*
8234	* Handle the VM-exit.
8235	*/
8236	rcStrict = vmxHCHandleExitNested(pVCpu, &VmxTransient);
8237	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitHandling, x);
8238	if (rcStrict == VINF_SUCCESS)
8239	{
8240	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8241	{
8242	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchNstGstVmexit);
8243	rcStrict = VINF_VMX_VMEXIT;
8244	}
8245	else
8246	{
8247	if (++(*pcLoops) <= cMaxResumeLoops)
8248	continue;
8249	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchMaxResumeLoops);
8250	rcStrict = VINF_EM_RAW_INTERRUPT;
8251	}
8252	}
8253	else
8254	Assert(rcStrict != VINF_VMX_VMEXIT);
8255	break;
8256	}
8257
8258	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatEntry, x);
8259	return rcStrict;
8260	}
8261	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
8262
8263
8264	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8265	* probes.
8266	*
8267	* The following few functions and associated structure contains the bloat
8268	* necessary for providing detailed debug events and dtrace probes as well as
8269	* reliable host side single stepping. This works on the principle of
8270	* "subclassing" the normal execution loop and workers. We replace the loop
8271	* method completely and override selected helpers to add necessary adjustments
8272	* to their core operation.
8273	*
8274	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8275	* any performance for debug and analysis features.
8276	*
8277	* @{
8278	*/
8279
8280	/**
8281	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
8282	* the debug run loop.
8283	*/
8284	typedef struct VMXRUNDBGSTATE
8285	{
8286	/** The RIP we started executing at. This is for detecting that we stepped. */
8287	uint64_t uRipStart;
8288	/** The CS we started executing with. */
8289	uint16_t uCsStart;
8290
8291	/** Whether we've actually modified the 1st execution control field. */
8292	bool fModifiedProcCtls : 1;
8293	/** Whether we've actually modified the 2nd execution control field. */
8294	bool fModifiedProcCtls2 : 1;
8295	/** Whether we've actually modified the exception bitmap. */
8296	bool fModifiedXcptBitmap : 1;
8297
8298	/** We desire the modified the CR0 mask to be cleared. */
8299	bool fClearCr0Mask : 1;
8300	/** We desire the modified the CR4 mask to be cleared. */
8301	bool fClearCr4Mask : 1;
8302	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8303	uint32_t fCpe1Extra;
8304	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8305	uint32_t fCpe1Unwanted;
8306	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8307	uint32_t fCpe2Extra;
8308	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
8309	uint32_t bmXcptExtra;
8310	/** The sequence number of the Dtrace provider settings the state was
8311	* configured against. */
8312	uint32_t uDtraceSettingsSeqNo;
8313	/** VM-exits to check (one bit per VM-exit). */
8314	uint32_t bmExitsToCheck[3];
8315
8316	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8317	uint32_t fProcCtlsInitial;
8318	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8319	uint32_t fProcCtls2Initial;
8320	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8321	uint32_t bmXcptInitial;
8322	} VMXRUNDBGSTATE;
8323	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
8324	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
8325
8326
8327	/**
8328	* Initializes the VMXRUNDBGSTATE structure.
8329	*
8330	* @param pVCpu The cross context virtual CPU structure of the
8331	* calling EMT.
8332	* @param pVmxTransient The VMX-transient structure.
8333	* @param pDbgState The debug state to initialize.
8334	*/
8335	static void vmxHCRunDebugStateInit(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
8336	{
8337	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
8338	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
8339
8340	pDbgState->fModifiedProcCtls = false;
8341	pDbgState->fModifiedProcCtls2 = false;
8342	pDbgState->fModifiedXcptBitmap = false;
8343	pDbgState->fClearCr0Mask = false;
8344	pDbgState->fClearCr4Mask = false;
8345	pDbgState->fCpe1Extra = 0;
8346	pDbgState->fCpe1Unwanted = 0;
8347	pDbgState->fCpe2Extra = 0;
8348	pDbgState->bmXcptExtra = 0;
8349	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
8350	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
8351	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
8352	}
8353
8354
8355	/**
8356	* Updates the VMSC fields with changes requested by @a pDbgState.
8357	*
8358	* This is performed after vmxHCPreRunGuestDebugStateUpdate as well
8359	* immediately before executing guest code, i.e. when interrupts are disabled.
8360	* We don't check status codes here as we cannot easily assert or return in the
8361	* latter case.
8362	*
8363	* @param pVCpu The cross context virtual CPU structure.
8364	* @param pVmxTransient The VMX-transient structure.
8365	* @param pDbgState The debug state.
8366	*/
8367	static void vmxHCPreRunGuestDebugStateApply(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
8368	{
8369	/*
8370	* Ensure desired flags in VMCS control fields are set.
8371	* (Ignoring write failure here, as we're committed and it's just debug extras.)
8372	*
8373	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
8374	* there should be no stale data in pCtx at this point.
8375	*/
8376	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8377	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
8378	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
8379	{
8380	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
8381	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
8382	VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8383	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
8384	pDbgState->fModifiedProcCtls = true;
8385	}
8386
8387	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
8388	{
8389	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
8390	VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
8391	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
8392	pDbgState->fModifiedProcCtls2 = true;
8393	}
8394
8395	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
8396	{
8397	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
8398	VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
8399	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
8400	pDbgState->fModifiedXcptBitmap = true;
8401	}
8402
8403	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
8404	{
8405	pVmcsInfo->u64Cr0Mask = 0;
8406	VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_MASK, 0);
8407	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
8408	}
8409
8410	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
8411	{
8412	pVmcsInfo->u64Cr4Mask = 0;
8413	VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR4_MASK, 0);
8414	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
8415	}
8416
8417	NOREF(pVCpu);
8418	}
8419
8420
8421	/**
8422	* Restores VMCS fields that were changed by vmxHCPreRunGuestDebugStateApply for
8423	* re-entry next time around.
8424	*
8425	* @returns Strict VBox status code (i.e. informational status codes too).
8426	* @param pVCpu The cross context virtual CPU structure.
8427	* @param pVmxTransient The VMX-transient structure.
8428	* @param pDbgState The debug state.
8429	* @param rcStrict The return code from executing the guest using single
8430	* stepping.
8431	*/
8432	static VBOXSTRICTRC vmxHCRunDebugStateRevert(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
8433	VBOXSTRICTRC rcStrict)
8434	{
8435	/*
8436	* Restore VM-exit control settings as we may not reenter this function the
8437	* next time around.
8438	*/
8439	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8440
8441	/* We reload the initial value, trigger what we can of recalculations the
8442	next time around. From the looks of things, that's all that's required atm. */
8443	if (pDbgState->fModifiedProcCtls)
8444	{
8445	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
8446	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in vmxHCLeave */
8447	int rc2 = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
8448	AssertRC(rc2);
8449	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
8450	}
8451
8452	/* We're currently the only ones messing with this one, so just restore the
8453	cached value and reload the field. */
8454	if ( pDbgState->fModifiedProcCtls2
8455	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
8456	{
8457	int rc2 = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
8458	AssertRC(rc2);
8459	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
8460	}
8461
8462	/* If we've modified the exception bitmap, we restore it and trigger
8463	reloading and partial recalculation the next time around. */
8464	if (pDbgState->fModifiedXcptBitmap)
8465	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
8466
8467	return rcStrict;
8468	}
8469
8470
8471	/**
8472	* Configures VM-exit controls for current DBGF and DTrace settings.
8473	*
8474	* This updates @a pDbgState and the VMCS execution control fields to reflect
8475	* the necessary VM-exits demanded by DBGF and DTrace.
8476	*
8477	* @param pVCpu The cross context virtual CPU structure.
8478	* @param pVmxTransient The VMX-transient structure. May update
8479	* fUpdatedTscOffsettingAndPreemptTimer.
8480	* @param pDbgState The debug state.
8481	*/
8482	static void vmxHCPreRunGuestDebugStateUpdate(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
8483	{
8484	/*
8485	* Take down the dtrace serial number so we can spot changes.
8486	*/
8487	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
8488	ASMCompilerBarrier();
8489
8490	/*
8491	* We'll rebuild most of the middle block of data members (holding the
8492	* current settings) as we go along here, so start by clearing it all.
8493	*/
8494	pDbgState->bmXcptExtra = 0;
8495	pDbgState->fCpe1Extra = 0;
8496	pDbgState->fCpe1Unwanted = 0;
8497	pDbgState->fCpe2Extra = 0;
8498	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
8499	pDbgState->bmExitsToCheck[i] = 0;
8500
8501	/*
8502	* Software interrupts (INT XXh) - no idea how to trigger these...
8503	*/
8504	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8505	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
8506	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
8507	{
8508	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8509	}
8510
8511	/*
8512	* INT3 breakpoints - triggered by #BP exceptions.
8513	*/
8514	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
8515	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
8516
8517	/*
8518	* Exception bitmap and XCPT events+probes.
8519	*/
8520	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
8521	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
8522	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
8523
8524	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
8525	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
8526	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
8527	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
8528	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
8529	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
8530	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
8531	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
8532	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
8533	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
8534	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
8535	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
8536	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
8537	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
8538	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
8539	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
8540	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
8541	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
8542
8543	if (pDbgState->bmXcptExtra)
8544	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8545
8546	/*
8547	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
8548	*
8549	* Note! This is the reverse of what vmxHCHandleExitDtraceEvents does.
8550	* So, when adding/changing/removing please don't forget to update it.
8551	*
8552	* Some of the macros are picking up local variables to save horizontal space,
8553	* (being able to see it in a table is the lesser evil here).
8554	*/
8555	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
8556	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
8557	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
8558	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
8559	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8560	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
8561	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
8562	} else do { } while (0)
8563	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
8564	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8565	{ \
8566	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
8567	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
8568	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
8569	} else do { } while (0)
8570	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
8571	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8572	{ \
8573	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
8574	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
8575	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
8576	} else do { } while (0)
8577	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
8578	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8579	{ \
8580	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
8581	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
8582	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
8583	} else do { } while (0)
8584
8585	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
8586	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
8587	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
8588	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
8589	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
8590
8591	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
8592	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
8593	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
8594	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
8595	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
8596	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
8597	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
8598	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
8599	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
8600	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
8601	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
8602	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
8603	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
8604	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
8605	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
8606	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
8607	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
8608	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
8609	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
8610	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
8611	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
8612	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
8613	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
8614	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
8615	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
8616	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
8617	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
8618	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
8619	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
8620	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
8621	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
8622	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
8623	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
8624	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
8625	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
8626	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
8627
8628	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
8629	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
8630	{
8631	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
8632	\| CPUMCTX_EXTRN_APIC_TPR);
8633	AssertRC(rc);
8634
8635	#if 0 /** @todo fix me */
8636	pDbgState->fClearCr0Mask = true;
8637	pDbgState->fClearCr4Mask = true;
8638	#endif
8639	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
8640	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
8641	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
8642	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
8643	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
8644	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
8645	require clearing here and in the loop if we start using it. */
8646	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
8647	}
8648	else
8649	{
8650	if (pDbgState->fClearCr0Mask)
8651	{
8652	pDbgState->fClearCr0Mask = false;
8653	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_CR0);
8654	}
8655	if (pDbgState->fClearCr4Mask)
8656	{
8657	pDbgState->fClearCr4Mask = false;
8658	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_CR4);
8659	}
8660	}
8661	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
8662	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
8663
8664	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
8665	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
8666	{
8667	/** @todo later, need to fix handler as it assumes this won't usually happen. */
8668	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
8669	}
8670	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
8671	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
8672
8673	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
8674	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
8675	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
8676	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
8677	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
8678	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
8679	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
8680	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
8681	#if 0 /** @todo too slow, fix handler. */
8682	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
8683	#endif
8684	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
8685
8686	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
8687	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
8688	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
8689	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
8690	{
8691	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
8692	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
8693	}
8694	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
8695	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
8696	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
8697	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
8698
8699	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
8700	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
8701	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
8702	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
8703	{
8704	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
8705	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
8706	}
8707	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
8708	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
8709	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
8710	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
8711
8712	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
8713	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
8714	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
8715	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
8716	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
8717	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
8718	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
8719	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
8720	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
8721	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
8722	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
8723	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
8724	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
8725	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
8726	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
8727	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
8728	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
8729	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
8730	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
8731	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
8732	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
8733	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
8734
8735	#undef IS_EITHER_ENABLED
8736	#undef SET_ONLY_XBM_IF_EITHER_EN
8737	#undef SET_CPE1_XBM_IF_EITHER_EN
8738	#undef SET_CPEU_XBM_IF_EITHER_EN
8739	#undef SET_CPE2_XBM_IF_EITHER_EN
8740
8741	/*
8742	* Sanitize the control stuff.
8743	*/
8744	pDbgState->fCpe2Extra &= g_HmMsrs.u.vmx.ProcCtls2.n.allowed1;
8745	if (pDbgState->fCpe2Extra)
8746	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
8747	pDbgState->fCpe1Extra &= g_HmMsrs.u.vmx.ProcCtls.n.allowed1;
8748	pDbgState->fCpe1Unwanted &= ~g_HmMsrs.u.vmx.ProcCtls.n.allowed0;
8749	if (pVCpu->hmr0.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
8750	{
8751	pVCpu->hmr0.s.fDebugWantRdTscExit ^= true;
8752	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
8753	}
8754
8755	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
8756	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
8757	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
8758	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
8759	}
8760
8761
8762	/**
8763	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
8764	* appropriate.
8765	*
8766	* The caller has checked the VM-exit against the
8767	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
8768	* already, so we don't have to do that either.
8769	*
8770	* @returns Strict VBox status code (i.e. informational status codes too).
8771	* @param pVCpu The cross context virtual CPU structure.
8772	* @param pVmxTransient The VMX-transient structure.
8773	* @param uExitReason The VM-exit reason.
8774	*
8775	* @remarks The name of this function is displayed by dtrace, so keep it short
8776	* and to the point. No longer than 33 chars long, please.
8777	*/
8778	static VBOXSTRICTRC vmxHCHandleExitDtraceEvents(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
8779	{
8780	/*
8781	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
8782	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
8783	*
8784	* Note! This is the reverse operation of what vmxHCPreRunGuestDebugStateUpdate
8785	* does. Must add/change/remove both places. Same ordering, please.
8786	*
8787	* Added/removed events must also be reflected in the next section
8788	* where we dispatch dtrace events.
8789	*/
8790	bool fDtrace1 = false;
8791	bool fDtrace2 = false;
8792	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
8793	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
8794	uint32_t uEventArg = 0;
8795	#define SET_EXIT(a_EventSubName) \
8796	do { \
8797	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
8798	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
8799	} while (0)
8800	#define SET_BOTH(a_EventSubName) \
8801	do { \
8802	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
8803	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
8804	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
8805	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
8806	} while (0)
8807	switch (uExitReason)
8808	{
8809	case VMX_EXIT_MTF:
8810	return vmxHCExitMtf(pVCpu, pVmxTransient);
8811
8812	case VMX_EXIT_XCPT_OR_NMI:
8813	{
8814	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
8815	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
8816	{
8817	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
8818	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
8819	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
8820	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
8821	{
8822	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
8823	{
8824	vmxHCReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
8825	uEventArg = pVmxTransient->uExitIntErrorCode;
8826	}
8827	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
8828	switch (enmEvent1)
8829	{
8830	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
8831	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
8832	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
8833	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
8834	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
8835	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
8836	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
8837	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
8838	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
8839	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
8840	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
8841	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
8842	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
8843	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
8844	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
8845	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
8846	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
8847	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
8848	default: break;
8849	}
8850	}
8851	else
8852	AssertFailed();
8853	break;
8854
8855	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
8856	uEventArg = idxVector;
8857	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
8858	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
8859	break;
8860	}
8861	break;
8862	}
8863
8864	case VMX_EXIT_TRIPLE_FAULT:
8865	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
8866	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
8867	break;
8868	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
8869	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
8870	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
8871	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
8872	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
8873
8874	/* Instruction specific VM-exits: */
8875	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
8876	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
8877	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
8878	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
8879	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
8880	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
8881	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
8882	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
8883	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
8884	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
8885	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
8886	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
8887	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
8888	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
8889	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
8890	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
8891	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
8892	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
8893	case VMX_EXIT_MOV_CRX:
8894	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
8895	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
8896	SET_BOTH(CRX_READ);
8897	else
8898	SET_BOTH(CRX_WRITE);
8899	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
8900	break;
8901	case VMX_EXIT_MOV_DRX:
8902	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
8903	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
8904	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
8905	SET_BOTH(DRX_READ);
8906	else
8907	SET_BOTH(DRX_WRITE);
8908	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
8909	break;
8910	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
8911	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
8912	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
8913	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
8914	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
8915	case VMX_EXIT_GDTR_IDTR_ACCESS:
8916	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
8917	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
8918	{
8919	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
8920	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
8921	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
8922	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
8923	}
8924	break;
8925
8926	case VMX_EXIT_LDTR_TR_ACCESS:
8927	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
8928	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
8929	{
8930	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
8931	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
8932	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
8933	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
8934	}
8935	break;
8936
8937	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
8938	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
8939	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
8940	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
8941	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
8942	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
8943	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
8944	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
8945	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
8946	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
8947	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
8948
8949	/* Events that aren't relevant at this point. */
8950	case VMX_EXIT_EXT_INT:
8951	case VMX_EXIT_INT_WINDOW:
8952	case VMX_EXIT_NMI_WINDOW:
8953	case VMX_EXIT_TPR_BELOW_THRESHOLD:
8954	case VMX_EXIT_PREEMPT_TIMER:
8955	case VMX_EXIT_IO_INSTR:
8956	break;
8957
8958	/* Errors and unexpected events. */
8959	case VMX_EXIT_INIT_SIGNAL:
8960	case VMX_EXIT_SIPI:
8961	case VMX_EXIT_IO_SMI:
8962	case VMX_EXIT_SMI:
8963	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
8964	case VMX_EXIT_ERR_MSR_LOAD:
8965	case VMX_EXIT_ERR_MACHINE_CHECK:
8966	case VMX_EXIT_PML_FULL:
8967	case VMX_EXIT_VIRTUALIZED_EOI:
8968	break;
8969
8970	default:
8971	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
8972	break;
8973	}
8974	#undef SET_BOTH
8975	#undef SET_EXIT
8976
8977	/*
8978	* Dtrace tracepoints go first. We do them here at once so we don't
8979	* have to copy the guest state saving and stuff a few dozen times.
8980	* Down side is that we've got to repeat the switch, though this time
8981	* we use enmEvent since the probes are a subset of what DBGF does.
8982	*/
8983	if (fDtrace1 \|\| fDtrace2)
8984	{
8985	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
8986	vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8987	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8988	switch (enmEvent1)
8989	{
8990	/** @todo consider which extra parameters would be helpful for each probe. */
8991	case DBGFEVENT_END: break;
8992	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
8993	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
8994	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
8995	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
8996	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
8997	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
8998	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
8999	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
9000	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
9001	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
9002	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
9003	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
9004	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
9005	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
9006	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
9007	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
9008	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
9009	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
9010	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9011	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9012	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
9013	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
9014	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
9015	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
9016	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
9017	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
9018	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
9019	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9020	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9021	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9022	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9023	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9024	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
9025	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9026	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
9027	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
9028	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
9029	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
9030	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
9031	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
9032	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
9033	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
9034	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
9035	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
9036	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
9037	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
9038	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
9039	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
9040	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
9041	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
9042	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
9043	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
9044	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
9045	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
9046	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
9047	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
9048	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
9049	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
9050	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
9051	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
9052	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
9053	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
9054	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
9055	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
9056	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
9057	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
9058	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9059	}
9060	switch (enmEvent2)
9061	{
9062	/** @todo consider which extra parameters would be helpful for each probe. */
9063	case DBGFEVENT_END: break;
9064	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
9065	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9066	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
9067	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
9068	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
9069	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
9070	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
9071	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
9072	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
9073	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9074	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9075	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9076	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9077	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9078	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
9079	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9080	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
9081	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
9082	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
9083	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
9084	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
9085	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
9086	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
9087	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
9088	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
9089	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
9090	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
9091	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
9092	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
9093	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
9094	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
9095	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
9096	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
9097	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
9098	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
9099	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
9100	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
9101	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
9102	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
9103	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
9104	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
9105	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
9106	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
9107	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
9108	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
9109	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
9110	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
9111	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
9112	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
9113	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
9114	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
9115	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
9116	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9117	}
9118	}
9119
9120	/*
9121	* Fire of the DBGF event, if enabled (our check here is just a quick one,
9122	* the DBGF call will do a full check).
9123	*
9124	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9125	* Note! If we have to events, we prioritize the first, i.e. the instruction
9126	* one, in order to avoid event nesting.
9127	*/
9128	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
9129	if ( enmEvent1 != DBGFEVENT_END
9130	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9131	{
9132	vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
9133	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
9134	if (rcStrict != VINF_SUCCESS)
9135	return rcStrict;
9136	}
9137	else if ( enmEvent2 != DBGFEVENT_END
9138	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9139	{
9140	vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
9141	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
9142	if (rcStrict != VINF_SUCCESS)
9143	return rcStrict;
9144	}
9145
9146	return VINF_SUCCESS;
9147	}
9148
9149
9150	/**
9151	* Single-stepping VM-exit filtering.
9152	*
9153	* This is preprocessing the VM-exits and deciding whether we've gotten far
9154	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9155	* handling is performed.
9156	*
9157	* @returns Strict VBox status code (i.e. informational status codes too).
9158	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
9159	* @param pVmxTransient The VMX-transient structure.
9160	* @param pDbgState The debug state.
9161	*/
9162	DECLINLINE(VBOXSTRICTRC) vmxHCRunDebugHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
9163	{
9164	/*
9165	* Expensive (saves context) generic dtrace VM-exit probe.
9166	*/
9167	uint32_t const uExitReason = pVmxTransient->uExitReason;
9168	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9169	{ /* more likely */ }
9170	else
9171	{
9172	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
9173	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
9174	AssertRC(rc);
9175	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
9176	}
9177
9178	#ifdef IN_RING0 /* NMIs should never reach R3. */
9179	/*
9180	* Check for host NMI, just to get that out of the way.
9181	*/
9182	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9183	{ /* normally likely */ }
9184	else
9185	{
9186	vmxHCReadExitIntInfoVmcs(pVCpu, pVmxTransient);
9187	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
9188	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
9189	return vmxHCExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
9190	}
9191	#endif
9192
9193	/*
9194	* Check for single stepping event if we're stepping.
9195	*/
9196	if (VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
9197	{
9198	switch (uExitReason)
9199	{
9200	case VMX_EXIT_MTF:
9201	return vmxHCExitMtf(pVCpu, pVmxTransient);
9202
9203	/* Various events: */
9204	case VMX_EXIT_XCPT_OR_NMI:
9205	case VMX_EXIT_EXT_INT:
9206	case VMX_EXIT_TRIPLE_FAULT:
9207	case VMX_EXIT_INT_WINDOW:
9208	case VMX_EXIT_NMI_WINDOW:
9209	case VMX_EXIT_TASK_SWITCH:
9210	case VMX_EXIT_TPR_BELOW_THRESHOLD:
9211	case VMX_EXIT_APIC_ACCESS:
9212	case VMX_EXIT_EPT_VIOLATION:
9213	case VMX_EXIT_EPT_MISCONFIG:
9214	case VMX_EXIT_PREEMPT_TIMER:
9215
9216	/* Instruction specific VM-exits: */
9217	case VMX_EXIT_CPUID:
9218	case VMX_EXIT_GETSEC:
9219	case VMX_EXIT_HLT:
9220	case VMX_EXIT_INVD:
9221	case VMX_EXIT_INVLPG:
9222	case VMX_EXIT_RDPMC:
9223	case VMX_EXIT_RDTSC:
9224	case VMX_EXIT_RSM:
9225	case VMX_EXIT_VMCALL:
9226	case VMX_EXIT_VMCLEAR:
9227	case VMX_EXIT_VMLAUNCH:
9228	case VMX_EXIT_VMPTRLD:
9229	case VMX_EXIT_VMPTRST:
9230	case VMX_EXIT_VMREAD:
9231	case VMX_EXIT_VMRESUME:
9232	case VMX_EXIT_VMWRITE:
9233	case VMX_EXIT_VMXOFF:
9234	case VMX_EXIT_VMXON:
9235	case VMX_EXIT_MOV_CRX:
9236	case VMX_EXIT_MOV_DRX:
9237	case VMX_EXIT_IO_INSTR:
9238	case VMX_EXIT_RDMSR:
9239	case VMX_EXIT_WRMSR:
9240	case VMX_EXIT_MWAIT:
9241	case VMX_EXIT_MONITOR:
9242	case VMX_EXIT_PAUSE:
9243	case VMX_EXIT_GDTR_IDTR_ACCESS:
9244	case VMX_EXIT_LDTR_TR_ACCESS:
9245	case VMX_EXIT_INVEPT:
9246	case VMX_EXIT_RDTSCP:
9247	case VMX_EXIT_INVVPID:
9248	case VMX_EXIT_WBINVD:
9249	case VMX_EXIT_XSETBV:
9250	case VMX_EXIT_RDRAND:
9251	case VMX_EXIT_INVPCID:
9252	case VMX_EXIT_VMFUNC:
9253	case VMX_EXIT_RDSEED:
9254	case VMX_EXIT_XSAVES:
9255	case VMX_EXIT_XRSTORS:
9256	{
9257	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
9258	AssertRCReturn(rc, rc);
9259	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
9260	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
9261	return VINF_EM_DBG_STEPPED;
9262	break;
9263	}
9264
9265	/* Errors and unexpected events: */
9266	case VMX_EXIT_INIT_SIGNAL:
9267	case VMX_EXIT_SIPI:
9268	case VMX_EXIT_IO_SMI:
9269	case VMX_EXIT_SMI:
9270	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9271	case VMX_EXIT_ERR_MSR_LOAD:
9272	case VMX_EXIT_ERR_MACHINE_CHECK:
9273	case VMX_EXIT_PML_FULL:
9274	case VMX_EXIT_VIRTUALIZED_EOI:
9275	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9276	break;
9277
9278	default:
9279	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9280	break;
9281	}
9282	}
9283
9284	/*
9285	* Check for debugger event breakpoints and dtrace probes.
9286	*/
9287	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9288	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9289	{
9290	VBOXSTRICTRC rcStrict = vmxHCHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
9291	if (rcStrict != VINF_SUCCESS)
9292	return rcStrict;
9293	}
9294
9295	/*
9296	* Normal processing.
9297	*/
9298	#ifdef HMVMX_USE_FUNCTION_TABLE
9299	return g_aVMExitHandlers[uExitReason].pfn(pVCpu, pVmxTransient);
9300	#else
9301	return vmxHCHandleExit(pVCpu, pVmxTransient, uExitReason);
9302	#endif
9303	}
9304
9305
9306	/**
9307	* Single steps guest code using hardware-assisted VMX.
9308	*
9309	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
9310	* but single-stepping through the hypervisor debugger.
9311	*
9312	* @returns Strict VBox status code (i.e. informational status codes too).
9313	* @param pVCpu The cross context virtual CPU structure.
9314	* @param pcLoops Pointer to the number of executed loops.
9315	*
9316	* @note Mostly the same as vmxHCRunGuestCodeNormal().
9317	*/
9318	static VBOXSTRICTRC vmxHCRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops)
9319	{
9320	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
9321	Assert(pcLoops);
9322	Assert(*pcLoops <= cMaxResumeLoops);
9323
9324	VMXTRANSIENT VmxTransient;
9325	RT_ZERO(VmxTransient);
9326	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9327
9328	/* Set HMCPU indicators. */
9329	bool const fSavedSingleInstruction = VCPU_2_VMXSTATE(pVCpu).fSingleInstruction;
9330	VCPU_2_VMXSTATE(pVCpu).fSingleInstruction = VCPU_2_VMXSTATE(pVCpu).fSingleInstruction \|\| DBGFIsStepping(pVCpu);
9331	pVCpu->hmr0.s.fDebugWantRdTscExit = false;
9332	pVCpu->hmr0.s.fUsingDebugLoop = true;
9333
9334	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9335	VMXRUNDBGSTATE DbgState;
9336	vmxHCRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
9337	vmxHCPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
9338
9339	/*
9340	* The loop.
9341	*/
9342	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9343	for (;;)
9344	{
9345	Assert(!HMR0SuspendPending());
9346	HMVMX_ASSERT_CPU_SAFE(pVCpu);
9347	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatEntry, x);
9348	bool fStepping = VCPU_2_VMXSTATE(pVCpu).fSingleInstruction;
9349
9350	/* Set up VM-execution controls the next two can respond to. */
9351	vmxHCPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
9352
9353	/*
9354	* Preparatory work for running guest code, this may force us to
9355	* return to ring-3.
9356	*
9357	* Warning! This bugger disables interrupts on VINF_SUCCESS!
9358	*/
9359	rcStrict = vmxHCPreRunGuest(pVCpu, &VmxTransient, fStepping);
9360	if (rcStrict != VINF_SUCCESS)
9361	break;
9362
9363	/* Interrupts are disabled at this point! */
9364	vmxHCPreRunGuestCommitted(pVCpu, &VmxTransient);
9365
9366	/* Override any obnoxious code in the above two calls. */
9367	vmxHCPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
9368
9369	/*
9370	* Finally execute the guest.
9371	*/
9372	int rcRun = vmxHCRunGuest(pVCpu, &VmxTransient);
9373
9374	vmxHCPostRunGuest(pVCpu, &VmxTransient, rcRun);
9375	/* Interrupts are re-enabled at this point! */
9376
9377	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9378	if (RT_SUCCESS(rcRun))
9379	{ /* very likely */ }
9380	else
9381	{
9382	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatPreExit, x);
9383	vmxHCReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
9384	return rcRun;
9385	}
9386
9387	/* Profile the VM-exit. */
9388	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9389	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitAll);
9390	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).aStatExitReason[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9391	STAM_PROFILE_ADV_STOP_START(&VCPU_2_VMXSTATS(pVCpu).StatPreExit, &VCPU_2_VMXSTATS(pVCpu).StatExitHandling, x);
9392	HMVMX_START_EXIT_DISPATCH_PROF();
9393
9394	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
9395
9396	/*
9397	* Handle the VM-exit - we quit earlier on certain VM-exits, see vmxHCHandleExitDebug().
9398	*/
9399	rcStrict = vmxHCRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
9400	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitHandling, x);
9401	if (rcStrict != VINF_SUCCESS)
9402	break;
9403	if (++(*pcLoops) > cMaxResumeLoops)
9404	{
9405	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchMaxResumeLoops);
9406	rcStrict = VINF_EM_RAW_INTERRUPT;
9407	break;
9408	}
9409
9410	/*
9411	* Stepping: Did the RIP change, if so, consider it a single step.
9412	* Otherwise, make sure one of the TFs gets set.
9413	*/
9414	if (fStepping)
9415	{
9416	int rc = vmxHCImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
9417	AssertRC(rc);
9418	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
9419	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
9420	{
9421	rcStrict = VINF_EM_DBG_STEPPED;
9422	break;
9423	}
9424	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_DR7);
9425	}
9426
9427	/*
9428	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
9429	*/
9430	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
9431	vmxHCPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
9432
9433	/* Restore all controls applied by vmxHCPreRunGuestDebugStateApply above. */
9434	rcStrict = vmxHCRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
9435	Assert(rcStrict == VINF_SUCCESS);
9436	}
9437
9438	/*
9439	* Clear the X86_EFL_TF if necessary.
9440	*/
9441	if (pVCpu->hmr0.s.fClearTrapFlag)
9442	{
9443	int rc = vmxHCImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
9444	AssertRC(rc);
9445	pVCpu->hmr0.s.fClearTrapFlag = false;
9446	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
9447	}
9448	/** @todo there seems to be issues with the resume flag when the monitor trap
9449	* flag is pending without being used. Seen early in bios init when
9450	* accessing APIC page in protected mode. */
9451
9452	/* Restore HMCPU indicators. */
9453	pVCpu->hmr0.s.fUsingDebugLoop = false;
9454	pVCpu->hmr0.s.fDebugWantRdTscExit = false;
9455	VCPU_2_VMXSTATE(pVCpu).fSingleInstruction = fSavedSingleInstruction;
9456
9457	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatEntry, x);
9458	return rcStrict;
9459	}
9460	#endif
9461
9462	/** @} */
9463
9464
9465	#ifndef HMVMX_USE_FUNCTION_TABLE
9466	/**
9467	* Handles a guest VM-exit from hardware-assisted VMX execution.
9468	*
9469	* @returns Strict VBox status code (i.e. informational status codes too).
9470	* @param pVCpu The cross context virtual CPU structure.
9471	* @param pVmxTransient The VMX-transient structure.
9472	*/
9473	DECLINLINE(VBOXSTRICTRC) vmxHCHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9474	{
9475	#ifdef DEBUG_ramshankar
9476	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
9477	do { \
9478	if (a_fSave != 0) \
9479	vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL); \
9480	VBOXSTRICTRC rcStrict = a_CallExpr; \
9481	if (a_fSave != 0) \
9482	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST); \
9483	return rcStrict; \
9484	} while (0)
9485	#else
9486	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
9487	#endif
9488	uint32_t const uExitReason = pVmxTransient->uExitReason;
9489	switch (uExitReason)
9490	{
9491	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, vmxHCExitEptMisconfig(pVCpu, pVmxTransient));
9492	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, vmxHCExitEptViolation(pVCpu, pVmxTransient));
9493	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, vmxHCExitIoInstr(pVCpu, pVmxTransient));
9494	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, vmxHCExitCpuid(pVCpu, pVmxTransient));
9495	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, vmxHCExitRdtsc(pVCpu, pVmxTransient));
9496	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, vmxHCExitRdtscp(pVCpu, pVmxTransient));
9497	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, vmxHCExitApicAccess(pVCpu, pVmxTransient));
9498	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, vmxHCExitXcptOrNmi(pVCpu, pVmxTransient));
9499	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, vmxHCExitMovCRx(pVCpu, pVmxTransient));
9500	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, vmxHCExitExtInt(pVCpu, pVmxTransient));
9501	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, vmxHCExitIntWindow(pVCpu, pVmxTransient));
9502	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, vmxHCExitTprBelowThreshold(pVCpu, pVmxTransient));
9503	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, vmxHCExitMwait(pVCpu, pVmxTransient));
9504	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, vmxHCExitMonitor(pVCpu, pVmxTransient));
9505	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, vmxHCExitTaskSwitch(pVCpu, pVmxTransient));
9506	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, vmxHCExitPreemptTimer(pVCpu, pVmxTransient));
9507	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, vmxHCExitRdmsr(pVCpu, pVmxTransient));
9508	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, vmxHCExitWrmsr(pVCpu, pVmxTransient));
9509	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, vmxHCExitVmcall(pVCpu, pVmxTransient));
9510	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, vmxHCExitMovDRx(pVCpu, pVmxTransient));
9511	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, vmxHCExitHlt(pVCpu, pVmxTransient));
9512	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, vmxHCExitInvd(pVCpu, pVmxTransient));
9513	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, vmxHCExitInvlpg(pVCpu, pVmxTransient));
9514	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, vmxHCExitMtf(pVCpu, pVmxTransient));
9515	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, vmxHCExitPause(pVCpu, pVmxTransient));
9516	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, vmxHCExitWbinvd(pVCpu, pVmxTransient));
9517	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, vmxHCExitXsetbv(pVCpu, pVmxTransient));
9518	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, vmxHCExitInvpcid(pVCpu, pVmxTransient));
9519	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, vmxHCExitGetsec(pVCpu, pVmxTransient));
9520	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, vmxHCExitRdpmc(pVCpu, pVmxTransient));
9521	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9522	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, vmxHCExitVmclear(pVCpu, pVmxTransient));
9523	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, vmxHCExitVmlaunch(pVCpu, pVmxTransient));
9524	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, vmxHCExitVmptrld(pVCpu, pVmxTransient));
9525	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, vmxHCExitVmptrst(pVCpu, pVmxTransient));
9526	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, vmxHCExitVmread(pVCpu, pVmxTransient));
9527	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, vmxHCExitVmwrite(pVCpu, pVmxTransient));
9528	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, vmxHCExitVmresume(pVCpu, pVmxTransient));
9529	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, vmxHCExitVmxoff(pVCpu, pVmxTransient));
9530	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, vmxHCExitVmxon(pVCpu, pVmxTransient));
9531	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, vmxHCExitInvvpid(pVCpu, pVmxTransient));
9532	case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, vmxHCExitSetPendingXcptUD(pVCpu, pVmxTransient));
9533	#else
9534	case VMX_EXIT_VMCLEAR:
9535	case VMX_EXIT_VMLAUNCH:
9536	case VMX_EXIT_VMPTRLD:
9537	case VMX_EXIT_VMPTRST:
9538	case VMX_EXIT_VMREAD:
9539	case VMX_EXIT_VMRESUME:
9540	case VMX_EXIT_VMWRITE:
9541	case VMX_EXIT_VMXOFF:
9542	case VMX_EXIT_VMXON:
9543	case VMX_EXIT_INVVPID:
9544	case VMX_EXIT_INVEPT:
9545	return vmxHCExitSetPendingXcptUD(pVCpu, pVmxTransient);
9546	#endif
9547
9548	case VMX_EXIT_TRIPLE_FAULT: return vmxHCExitTripleFault(pVCpu, pVmxTransient);
9549	case VMX_EXIT_NMI_WINDOW: return vmxHCExitNmiWindow(pVCpu, pVmxTransient);
9550	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return vmxHCExitErrInvalidGuestState(pVCpu, pVmxTransient);
9551
9552	case VMX_EXIT_INIT_SIGNAL:
9553	case VMX_EXIT_SIPI:
9554	case VMX_EXIT_IO_SMI:
9555	case VMX_EXIT_SMI:
9556	case VMX_EXIT_ERR_MSR_LOAD:
9557	case VMX_EXIT_ERR_MACHINE_CHECK:
9558	case VMX_EXIT_PML_FULL:
9559	case VMX_EXIT_VIRTUALIZED_EOI:
9560	case VMX_EXIT_GDTR_IDTR_ACCESS:
9561	case VMX_EXIT_LDTR_TR_ACCESS:
9562	case VMX_EXIT_APIC_WRITE:
9563	case VMX_EXIT_RDRAND:
9564	case VMX_EXIT_RSM:
9565	case VMX_EXIT_VMFUNC:
9566	case VMX_EXIT_ENCLS:
9567	case VMX_EXIT_RDSEED:
9568	case VMX_EXIT_XSAVES:
9569	case VMX_EXIT_XRSTORS:
9570	case VMX_EXIT_UMWAIT:
9571	case VMX_EXIT_TPAUSE:
9572	case VMX_EXIT_LOADIWKEY:
9573	default:
9574	return vmxHCExitErrUnexpected(pVCpu, pVmxTransient);
9575	}
9576	#undef VMEXIT_CALL_RET
9577	}
9578	#endif /* !HMVMX_USE_FUNCTION_TABLE */
9579
9580
9581	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9582	/**
9583	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
9584	*
9585	* @returns Strict VBox status code (i.e. informational status codes too).
9586	* @param pVCpu The cross context virtual CPU structure.
9587	* @param pVmxTransient The VMX-transient structure.
9588	*/
9589	DECLINLINE(VBOXSTRICTRC) vmxHCHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9590	{
9591	uint32_t const uExitReason = pVmxTransient->uExitReason;
9592	switch (uExitReason)
9593	{
9594	case VMX_EXIT_EPT_MISCONFIG: return vmxHCExitEptMisconfig(pVCpu, pVmxTransient);
9595	case VMX_EXIT_EPT_VIOLATION: return vmxHCExitEptViolation(pVCpu, pVmxTransient);
9596	case VMX_EXIT_XCPT_OR_NMI: return vmxHCExitXcptOrNmiNested(pVCpu, pVmxTransient);
9597	case VMX_EXIT_IO_INSTR: return vmxHCExitIoInstrNested(pVCpu, pVmxTransient);
9598	case VMX_EXIT_HLT: return vmxHCExitHltNested(pVCpu, pVmxTransient);
9599
9600	/*
9601	* We shouldn't direct host physical interrupts to the nested-guest.
9602	*/
9603	case VMX_EXIT_EXT_INT:
9604	return vmxHCExitExtInt(pVCpu, pVmxTransient);
9605
9606	/*
9607	* Instructions that cause VM-exits unconditionally or the condition is
9608	* always is taken solely from the nested hypervisor (meaning if the VM-exit
9609	* happens, it's guaranteed to be a nested-guest VM-exit).
9610	*
9611	* - Provides VM-exit instruction length ONLY.
9612	*/
9613	case VMX_EXIT_CPUID: /* Unconditional. */
9614	case VMX_EXIT_VMCALL:
9615	case VMX_EXIT_GETSEC:
9616	case VMX_EXIT_INVD:
9617	case VMX_EXIT_XSETBV:
9618	case VMX_EXIT_VMLAUNCH:
9619	case VMX_EXIT_VMRESUME:
9620	case VMX_EXIT_VMXOFF:
9621	case VMX_EXIT_ENCLS: /* Condition specified solely by nested hypervisor. */
9622	case VMX_EXIT_VMFUNC:
9623	return vmxHCExitInstrNested(pVCpu, pVmxTransient);
9624
9625	/*
9626	* Instructions that cause VM-exits unconditionally or the condition is
9627	* always is taken solely from the nested hypervisor (meaning if the VM-exit
9628	* happens, it's guaranteed to be a nested-guest VM-exit).
9629	*
9630	* - Provides VM-exit instruction length.
9631	* - Provides VM-exit information.
9632	* - Optionally provides Exit qualification.
9633	*
9634	* Since Exit qualification is 0 for all VM-exits where it is not
9635	* applicable, reading and passing it to the guest should produce
9636	* defined behavior.
9637	*
9638	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
9639	*/
9640	case VMX_EXIT_INVEPT: /* Unconditional. */
9641	case VMX_EXIT_INVVPID:
9642	case VMX_EXIT_VMCLEAR:
9643	case VMX_EXIT_VMPTRLD:
9644	case VMX_EXIT_VMPTRST:
9645	case VMX_EXIT_VMXON:
9646	case VMX_EXIT_GDTR_IDTR_ACCESS: /* Condition specified solely by nested hypervisor. */
9647	case VMX_EXIT_LDTR_TR_ACCESS:
9648	case VMX_EXIT_RDRAND:
9649	case VMX_EXIT_RDSEED:
9650	case VMX_EXIT_XSAVES:
9651	case VMX_EXIT_XRSTORS:
9652	case VMX_EXIT_UMWAIT:
9653	case VMX_EXIT_TPAUSE:
9654	return vmxHCExitInstrWithInfoNested(pVCpu, pVmxTransient);
9655
9656	case VMX_EXIT_RDTSC: return vmxHCExitRdtscNested(pVCpu, pVmxTransient);
9657	case VMX_EXIT_RDTSCP: return vmxHCExitRdtscpNested(pVCpu, pVmxTransient);
9658	case VMX_EXIT_RDMSR: return vmxHCExitRdmsrNested(pVCpu, pVmxTransient);
9659	case VMX_EXIT_WRMSR: return vmxHCExitWrmsrNested(pVCpu, pVmxTransient);
9660	case VMX_EXIT_INVLPG: return vmxHCExitInvlpgNested(pVCpu, pVmxTransient);
9661	case VMX_EXIT_INVPCID: return vmxHCExitInvpcidNested(pVCpu, pVmxTransient);
9662	case VMX_EXIT_TASK_SWITCH: return vmxHCExitTaskSwitchNested(pVCpu, pVmxTransient);
9663	case VMX_EXIT_WBINVD: return vmxHCExitWbinvdNested(pVCpu, pVmxTransient);
9664	case VMX_EXIT_MTF: return vmxHCExitMtfNested(pVCpu, pVmxTransient);
9665	case VMX_EXIT_APIC_ACCESS: return vmxHCExitApicAccessNested(pVCpu, pVmxTransient);
9666	case VMX_EXIT_APIC_WRITE: return vmxHCExitApicWriteNested(pVCpu, pVmxTransient);
9667	case VMX_EXIT_VIRTUALIZED_EOI: return vmxHCExitVirtEoiNested(pVCpu, pVmxTransient);
9668	case VMX_EXIT_MOV_CRX: return vmxHCExitMovCRxNested(pVCpu, pVmxTransient);
9669	case VMX_EXIT_INT_WINDOW: return vmxHCExitIntWindowNested(pVCpu, pVmxTransient);
9670	case VMX_EXIT_NMI_WINDOW: return vmxHCExitNmiWindowNested(pVCpu, pVmxTransient);
9671	case VMX_EXIT_TPR_BELOW_THRESHOLD: return vmxHCExitTprBelowThresholdNested(pVCpu, pVmxTransient);
9672	case VMX_EXIT_MWAIT: return vmxHCExitMwaitNested(pVCpu, pVmxTransient);
9673	case VMX_EXIT_MONITOR: return vmxHCExitMonitorNested(pVCpu, pVmxTransient);
9674	case VMX_EXIT_PAUSE: return vmxHCExitPauseNested(pVCpu, pVmxTransient);
9675
9676	case VMX_EXIT_PREEMPT_TIMER:
9677	{
9678	/** @todo NSTVMX: Preempt timer. */
9679	return vmxHCExitPreemptTimer(pVCpu, pVmxTransient);
9680	}
9681
9682	case VMX_EXIT_MOV_DRX: return vmxHCExitMovDRxNested(pVCpu, pVmxTransient);
9683	case VMX_EXIT_RDPMC: return vmxHCExitRdpmcNested(pVCpu, pVmxTransient);
9684
9685	case VMX_EXIT_VMREAD:
9686	case VMX_EXIT_VMWRITE: return vmxHCExitVmreadVmwriteNested(pVCpu, pVmxTransient);
9687
9688	case VMX_EXIT_TRIPLE_FAULT: return vmxHCExitTripleFaultNested(pVCpu, pVmxTransient);
9689	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return vmxHCExitErrInvalidGuestStateNested(pVCpu, pVmxTransient);
9690
9691	case VMX_EXIT_INIT_SIGNAL:
9692	case VMX_EXIT_SIPI:
9693	case VMX_EXIT_IO_SMI:
9694	case VMX_EXIT_SMI:
9695	case VMX_EXIT_ERR_MSR_LOAD:
9696	case VMX_EXIT_ERR_MACHINE_CHECK:
9697	case VMX_EXIT_PML_FULL:
9698	case VMX_EXIT_RSM:
9699	default:
9700	return vmxHCExitErrUnexpected(pVCpu, pVmxTransient);
9701	}
9702	}
9703	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
9704
9705
9706	/** @name VM-exit helpers.
9707	* @{
9708	*/
9709	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9710	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit helpers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9711	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9712
9713	/** Macro for VM-exits called unexpectedly. */
9714	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
9715	do { \
9716	VCPU_2_VMXSTATE((a_pVCpu)).u32HMError = (a_HmError); \
9717	return VERR_VMX_UNEXPECTED_EXIT; \
9718	} while (0)
9719
9720	#ifdef VBOX_STRICT
9721	# ifdef IN_RING0
9722	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
9723	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
9724	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
9725
9726	# define HMVMX_ASSERT_PREEMPT_CPUID() \
9727	do { \
9728	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
9729	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
9730	} while (0)
9731
9732	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
9733	do { \
9734	AssertPtr((a_pVCpu)); \
9735	AssertPtr((a_pVmxTransient)); \
9736	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
9737	Assert((a_pVmxTransient)->pVmcsInfo); \
9738	Assert(ASMIntAreEnabled()); \
9739	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
9740	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
9741	Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
9742	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
9743	if (!VMMRZCallRing3IsEnabled((a_pVCpu))) \
9744	HMVMX_ASSERT_PREEMPT_CPUID(); \
9745	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9746	} while (0)
9747	# else
9748	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() do { } while(0)
9749	# define HMVMX_ASSERT_PREEMPT_CPUID() do { } while(0)
9750	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
9751	do { \
9752	AssertPtr((a_pVCpu)); \
9753	AssertPtr((a_pVmxTransient)); \
9754	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
9755	Assert((a_pVmxTransient)->pVmcsInfo); \
9756	Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
9757	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9758	} while (0)
9759	# endif
9760
9761	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
9762	do { \
9763	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
9764	Assert((a_pVmxTransient)->fIsNestedGuest); \
9765	} while (0)
9766
9767	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
9768	do { \
9769	Log4Func(("\n")); \
9770	} while (0)
9771	#else
9772	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
9773	do { \
9774	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9775	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
9776	} while (0)
9777
9778	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
9779	do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
9780
9781	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
9782	#endif
9783
9784	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9785	/** Macro that does the necessary privilege checks and intercepted VM-exits for
9786	* guests that attempted to execute a VMX instruction. */
9787	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
9788	do \
9789	{ \
9790	VBOXSTRICTRC rcStrictTmp = vmxHCCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
9791	if (rcStrictTmp == VINF_SUCCESS) \
9792	{ /* likely */ } \
9793	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
9794	{ \
9795	Assert((a_pVCpu)->hm.s.Event.fPending); \
9796	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
9797	return VINF_SUCCESS; \
9798	} \
9799	else \
9800	{ \
9801	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
9802	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
9803	} \
9804	} while (0)
9805
9806	/** Macro that decodes a memory operand for an VM-exit caused by an instruction. */
9807	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
9808	do \
9809	{ \
9810	VBOXSTRICTRC rcStrictTmp = vmxHCDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
9811	(a_pGCPtrEffAddr)); \
9812	if (rcStrictTmp == VINF_SUCCESS) \
9813	{ /* likely */ } \
9814	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
9815	{ \
9816	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
9817	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
9818	NOREF(uXcptTmp); \
9819	return VINF_SUCCESS; \
9820	} \
9821	else \
9822	{ \
9823	Log4Func(("vmxHCDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
9824	return rcStrictTmp; \
9825	} \
9826	} while (0)
9827	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
9828
9829
9830	/**
9831	* Advances the guest RIP by the specified number of bytes.
9832	*
9833	* @param pVCpu The cross context virtual CPU structure.
9834	* @param cbInstr Number of bytes to advance the RIP by.
9835	*
9836	* @remarks No-long-jump zone!!!
9837	*/
9838	DECLINLINE(void) vmxHCAdvanceGuestRipBy(PVMCPUCC pVCpu, uint32_t cbInstr)
9839	{
9840	/* Advance the RIP. */
9841	pVCpu->cpum.GstCtx.rip += cbInstr;
9842	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP);
9843
9844	/* Update interrupt inhibition. */
9845	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
9846	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
9847	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
9848	}
9849
9850
9851	/**
9852	* Advances the guest RIP after reading it from the VMCS.
9853	*
9854	* @returns VBox status code, no informational status codes.
9855	* @param pVCpu The cross context virtual CPU structure.
9856	* @param pVmxTransient The VMX-transient structure.
9857	*
9858	* @remarks No-long-jump zone!!!
9859	*/
9860	static int vmxHCAdvanceGuestRip(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9861	{
9862	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9863	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
9864	AssertRCReturn(rc, rc);
9865
9866	vmxHCAdvanceGuestRipBy(pVCpu, pVmxTransient->cbExitInstr);
9867	return VINF_SUCCESS;
9868	}
9869
9870
9871	/**
9872	* Handle a condition that occurred while delivering an event through the guest or
9873	* nested-guest IDT.
9874	*
9875	* @returns Strict VBox status code (i.e. informational status codes too).
9876	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
9877	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
9878	* to continue execution of the guest which will delivery the \#DF.
9879	* @retval VINF_EM_RESET if we detected a triple-fault condition.
9880	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
9881	*
9882	* @param pVCpu The cross context virtual CPU structure.
9883	* @param pVmxTransient The VMX-transient structure.
9884	*
9885	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
9886	* Additionally, HMVMX_READ_EXIT_QUALIFICATION is required if the VM-exit
9887	* is due to an EPT violation, PML full or SPP-related event.
9888	*
9889	* @remarks No-long-jump zone!!!
9890	*/
9891	static VBOXSTRICTRC vmxHCCheckExitDueToEventDelivery(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9892	{
9893	Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending);
9894	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
9895	if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
9896	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
9897	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
9898	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_EXIT_QUALIFICATION);
9899
9900	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
9901	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9902	uint32_t const uIdtVectorInfo = pVmxTransient->uIdtVectoringInfo;
9903	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
9904	if (VMX_IDT_VECTORING_INFO_IS_VALID(uIdtVectorInfo))
9905	{
9906	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(uIdtVectorInfo);
9907	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(uIdtVectorInfo);
9908
9909	/*
9910	* If the event was a software interrupt (generated with INT n) or a software exception
9911	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
9912	* can handle the VM-exit and continue guest execution which will re-execute the
9913	* instruction rather than re-injecting the exception, as that can cause premature
9914	* trips to ring-3 before injection and involve TRPM which currently has no way of
9915	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
9916	* the problem).
9917	*/
9918	IEMXCPTRAISE enmRaise;
9919	IEMXCPTRAISEINFO fRaiseInfo;
9920	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
9921	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
9922	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
9923	{
9924	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
9925	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
9926	}
9927	else if (VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo))
9928	{
9929	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
9930	uint8_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
9931	Assert(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT);
9932
9933	uint32_t const fIdtVectorFlags = vmxHCGetIemXcptFlags(uIdtVector, uIdtVectorType);
9934	uint32_t const fExitVectorFlags = vmxHCGetIemXcptFlags(uExitVector, uExitVectorType);
9935
9936	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
9937
9938	/* Determine a vectoring #PF condition, see comment in vmxHCExitXcptPF(). */
9939	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
9940	{
9941	pVmxTransient->fVectoringPF = true;
9942	enmRaise = IEMXCPTRAISE_PREV_EVENT;
9943	}
9944	}
9945	else
9946	{
9947	/*
9948	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
9949	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
9950	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
9951	*/
9952	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
9953	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
9954	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
9955	enmRaise = IEMXCPTRAISE_PREV_EVENT;
9956	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
9957	}
9958
9959	/*
9960	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
9961	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
9962	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
9963	* subsequent VM-entry would fail, see @bugref{7445}.
9964	*
9965	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception".
9966	*/
9967	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
9968	&& enmRaise == IEMXCPTRAISE_PREV_EVENT
9969	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9970	&& CPUMIsGuestNmiBlocking(pVCpu))
9971	{
9972	CPUMSetGuestNmiBlocking(pVCpu, false);
9973	}
9974
9975	switch (enmRaise)
9976	{
9977	case IEMXCPTRAISE_CURRENT_XCPT:
9978	{
9979	Log4Func(("IDT: Pending secondary Xcpt: idtinfo=%#RX64 exitinfo=%#RX64\n", uIdtVectorInfo, uExitIntInfo));
9980	Assert(rcStrict == VINF_SUCCESS);
9981	break;
9982	}
9983
9984	case IEMXCPTRAISE_PREV_EVENT:
9985	{
9986	uint32_t u32ErrCode;
9987	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(uIdtVectorInfo))
9988	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
9989	else
9990	u32ErrCode = 0;
9991
9992	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see vmxHCExitXcptPF(). */
9993	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectReflect);
9994	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(uIdtVectorInfo), 0 /* cbInstr */,
9995	u32ErrCode, pVCpu->cpum.GstCtx.cr2);
9996
9997	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
9998	VCPU_2_VMXSTATE(pVCpu).Event.u32ErrCode));
9999	Assert(rcStrict == VINF_SUCCESS);
10000	break;
10001	}
10002
10003	case IEMXCPTRAISE_REEXEC_INSTR:
10004	Assert(rcStrict == VINF_SUCCESS);
10005	break;
10006
10007	case IEMXCPTRAISE_DOUBLE_FAULT:
10008	{
10009	/*
10010	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
10011	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
10012	*/
10013	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
10014	{
10015	pVmxTransient->fVectoringDoublePF = true;
10016	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
10017	pVCpu->cpum.GstCtx.cr2));
10018	rcStrict = VINF_SUCCESS;
10019	}
10020	else
10021	{
10022	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectConvertDF);
10023	vmxHCSetPendingXcptDF(pVCpu);
10024	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
10025	uIdtVector, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
10026	rcStrict = VINF_HM_DOUBLE_FAULT;
10027	}
10028	break;
10029	}
10030
10031	case IEMXCPTRAISE_TRIPLE_FAULT:
10032	{
10033	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector,
10034	VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
10035	rcStrict = VINF_EM_RESET;
10036	break;
10037	}
10038
10039	case IEMXCPTRAISE_CPU_HANG:
10040	{
10041	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
10042	rcStrict = VERR_EM_GUEST_CPU_HANG;
10043	break;
10044	}
10045
10046	default:
10047	{
10048	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
10049	rcStrict = VERR_VMX_IPE_2;
10050	break;
10051	}
10052	}
10053	}
10054	else if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
10055	&& !CPUMIsGuestNmiBlocking(pVCpu))
10056	{
10057	if ( VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo)
10058	&& VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo) != X86_XCPT_DF
10059	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(uExitIntInfo))
10060	{
10061	/*
10062	* Execution of IRET caused a fault when NMI blocking was in effect (i.e we're in
10063	* the guest or nested-guest NMI handler). We need to set the block-by-NMI field so
10064	* that virtual NMIs remain blocked until the IRET execution is completed.
10065	*
10066	* See Intel spec. 31.7.1.2 "Resuming Guest Software After Handling An Exception".
10067	*/
10068	CPUMSetGuestNmiBlocking(pVCpu, true);
10069	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
10070	}
10071	else if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
10072	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
10073	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
10074	{
10075	/*
10076	* Execution of IRET caused an EPT violation, page-modification log-full event or
10077	* SPP-related event VM-exit when NMI blocking was in effect (i.e. we're in the
10078	* guest or nested-guest NMI handler). We need to set the block-by-NMI field so
10079	* that virtual NMIs remain blocked until the IRET execution is completed.
10080	*
10081	* See Intel spec. 27.2.3 "Information about NMI unblocking due to IRET"
10082	*/
10083	if (VMX_EXIT_QUAL_EPT_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitQual))
10084	{
10085	CPUMSetGuestNmiBlocking(pVCpu, true);
10086	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
10087	}
10088	}
10089	}
10090
10091	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
10092	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
10093	return rcStrict;
10094	}
10095
10096
10097	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10098	/**
10099	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
10100	* guest attempting to execute a VMX instruction.
10101	*
10102	* @returns Strict VBox status code (i.e. informational status codes too).
10103	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
10104	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
10105	*
10106	* @param pVCpu The cross context virtual CPU structure.
10107	* @param uExitReason The VM-exit reason.
10108	*
10109	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
10110	* @remarks No-long-jump zone!!!
10111	*/
10112	static VBOXSTRICTRC vmxHCCheckExitDueToVmxInstr(PVMCPUCC pVCpu, uint32_t uExitReason)
10113	{
10114	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
10115	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
10116
10117	/*
10118	* The physical CPU would have already checked the CPU mode/code segment.
10119	* We shall just assert here for paranoia.
10120	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
10121	*/
10122	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
10123	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
10124	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
10125
10126	if (uExitReason == VMX_EXIT_VMXON)
10127	{
10128	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
10129
10130	/*
10131	* We check CR4.VMXE because it is required to be always set while in VMX operation
10132	* by physical CPUs and our CR4 read-shadow is only consulted when executing specific
10133	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
10134	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
10135	*/
10136	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
10137	{
10138	Log4Func(("CR4.VMXE is not set -> #UD\n"));
10139	vmxHCSetPendingXcptUD(pVCpu);
10140	return VINF_HM_PENDING_XCPT;
10141	}
10142	}
10143	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
10144	{
10145	/*
10146	* The guest has not entered VMX operation but attempted to execute a VMX instruction
10147	* (other than VMXON), we need to raise a #UD.
10148	*/
10149	Log4Func(("Not in VMX root mode -> #UD\n"));
10150	vmxHCSetPendingXcptUD(pVCpu);
10151	return VINF_HM_PENDING_XCPT;
10152	}
10153
10154	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
10155	return VINF_SUCCESS;
10156	}
10157
10158
10159	/**
10160	* Decodes the memory operand of an instruction that caused a VM-exit.
10161	*
10162	* The Exit qualification field provides the displacement field for memory
10163	* operand instructions, if any.
10164	*
10165	* @returns Strict VBox status code (i.e. informational status codes too).
10166	* @retval VINF_SUCCESS if the operand was successfully decoded.
10167	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
10168	* operand.
10169	* @param pVCpu The cross context virtual CPU structure.
10170	* @param uExitInstrInfo The VM-exit instruction information field.
10171	* @param enmMemAccess The memory operand's access type (read or write).
10172	* @param GCPtrDisp The instruction displacement field, if any. For
10173	* RIP-relative addressing pass RIP + displacement here.
10174	* @param pGCPtrMem Where to store the effective destination memory address.
10175	*
10176	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
10177	* virtual-8086 mode hence skips those checks while verifying if the
10178	* segment is valid.
10179	*/
10180	static VBOXSTRICTRC vmxHCDecodeMemOperand(PVMCPUCC pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
10181	PRTGCPTR pGCPtrMem)
10182	{
10183	Assert(pGCPtrMem);
10184	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
10185	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
10186	\| CPUMCTX_EXTRN_CR0);
10187
10188	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
10189	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
10190	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
10191
10192	VMXEXITINSTRINFO ExitInstrInfo;
10193	ExitInstrInfo.u = uExitInstrInfo;
10194	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
10195	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
10196	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
10197	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
10198	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
10199	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
10200	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
10201	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
10202	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
10203
10204	/*
10205	* Validate instruction information.
10206	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
10207	*/
10208	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
10209	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
10210	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
10211	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
10212	AssertLogRelMsgReturn(fIsMemOperand,
10213	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
10214
10215	/*
10216	* Compute the complete effective address.
10217	*
10218	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
10219	* See AMD spec. 4.5.2 "Segment Registers".
10220	*/
10221	RTGCPTR GCPtrMem = GCPtrDisp;
10222	if (fBaseRegValid)
10223	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
10224	if (fIdxRegValid)
10225	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
10226
10227	RTGCPTR const GCPtrOff = GCPtrMem;
10228	if ( !fIsLongMode
10229	\|\| iSegReg >= X86_SREG_FS)
10230	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
10231	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
10232
10233	/*
10234	* Validate effective address.
10235	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
10236	*/
10237	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
10238	Assert(cbAccess > 0);
10239	if (fIsLongMode)
10240	{
10241	if (X86_IS_CANONICAL(GCPtrMem))
10242	{
10243	*pGCPtrMem = GCPtrMem;
10244	return VINF_SUCCESS;
10245	}
10246
10247	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
10248	* "Data Limit Checks in 64-bit Mode". */
10249	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
10250	vmxHCSetPendingXcptGP(pVCpu, 0);
10251	return VINF_HM_PENDING_XCPT;
10252	}
10253
10254	/*
10255	* This is a watered down version of iemMemApplySegment().
10256	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
10257	* and segment CPL/DPL checks are skipped.
10258	*/
10259	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
10260	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
10261	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
10262
10263	/* Check if the segment is present and usable. */
10264	if ( pSel->Attr.n.u1Present
10265	&& !pSel->Attr.n.u1Unusable)
10266	{
10267	Assert(pSel->Attr.n.u1DescType);
10268	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
10269	{
10270	/* Check permissions for the data segment. */
10271	if ( enmMemAccess == VMXMEMACCESS_WRITE
10272	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
10273	{
10274	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
10275	vmxHCSetPendingXcptGP(pVCpu, iSegReg);
10276	return VINF_HM_PENDING_XCPT;
10277	}
10278
10279	/* Check limits if it's a normal data segment. */
10280	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
10281	{
10282	if ( GCPtrFirst32 > pSel->u32Limit
10283	\|\| GCPtrLast32 > pSel->u32Limit)
10284	{
10285	Log4Func(("Data segment limit exceeded. "
10286	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
10287	GCPtrLast32, pSel->u32Limit));
10288	if (iSegReg == X86_SREG_SS)
10289	vmxHCSetPendingXcptSS(pVCpu, 0);
10290	else
10291	vmxHCSetPendingXcptGP(pVCpu, 0);
10292	return VINF_HM_PENDING_XCPT;
10293	}
10294	}
10295	else
10296	{
10297	/* Check limits if it's an expand-down data segment.
10298	Note! The upper boundary is defined by the B bit, not the G bit! */
10299	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
10300	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
10301	{
10302	Log4Func(("Expand-down data segment limit exceeded. "
10303	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
10304	GCPtrLast32, pSel->u32Limit));
10305	if (iSegReg == X86_SREG_SS)
10306	vmxHCSetPendingXcptSS(pVCpu, 0);
10307	else
10308	vmxHCSetPendingXcptGP(pVCpu, 0);
10309	return VINF_HM_PENDING_XCPT;
10310	}
10311	}
10312	}
10313	else
10314	{
10315	/* Check permissions for the code segment. */
10316	if ( enmMemAccess == VMXMEMACCESS_WRITE
10317	\|\| ( enmMemAccess == VMXMEMACCESS_READ
10318	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
10319	{
10320	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
10321	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
10322	vmxHCSetPendingXcptGP(pVCpu, 0);
10323	return VINF_HM_PENDING_XCPT;
10324	}
10325
10326	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
10327	if ( GCPtrFirst32 > pSel->u32Limit
10328	\|\| GCPtrLast32 > pSel->u32Limit)
10329	{
10330	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
10331	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
10332	if (iSegReg == X86_SREG_SS)
10333	vmxHCSetPendingXcptSS(pVCpu, 0);
10334	else
10335	vmxHCSetPendingXcptGP(pVCpu, 0);
10336	return VINF_HM_PENDING_XCPT;
10337	}
10338	}
10339	}
10340	else
10341	{
10342	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
10343	vmxHCSetPendingXcptGP(pVCpu, 0);
10344	return VINF_HM_PENDING_XCPT;
10345	}
10346
10347	*pGCPtrMem = GCPtrMem;
10348	return VINF_SUCCESS;
10349	}
10350	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10351
10352
10353	/**
10354	* VM-exit helper for LMSW.
10355	*/
10356	static VBOXSTRICTRC vmxHCExitLmsw(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst)
10357	{
10358	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
10359	AssertRCReturn(rc, rc);
10360
10361	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
10362	AssertMsg( rcStrict == VINF_SUCCESS
10363	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10364
10365	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
10366	if (rcStrict == VINF_IEM_RAISED_XCPT)
10367	{
10368	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
10369	rcStrict = VINF_SUCCESS;
10370	}
10371
10372	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitLmsw);
10373	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10374	return rcStrict;
10375	}
10376
10377
10378	/**
10379	* VM-exit helper for CLTS.
10380	*/
10381	static VBOXSTRICTRC vmxHCExitClts(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
10382	{
10383	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
10384	AssertRCReturn(rc, rc);
10385
10386	VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
10387	AssertMsg( rcStrict == VINF_SUCCESS
10388	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10389
10390	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
10391	if (rcStrict == VINF_IEM_RAISED_XCPT)
10392	{
10393	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
10394	rcStrict = VINF_SUCCESS;
10395	}
10396
10397	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitClts);
10398	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10399	return rcStrict;
10400	}
10401
10402
10403	/**
10404	* VM-exit helper for MOV from CRx (CRx read).
10405	*/
10406	static VBOXSTRICTRC vmxHCExitMovFromCrX(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
10407	{
10408	Assert(iCrReg < 16);
10409	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
10410
10411	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
10412	AssertRCReturn(rc, rc);
10413
10414	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
10415	AssertMsg( rcStrict == VINF_SUCCESS
10416	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10417
10418	if (iGReg == X86_GREG_xSP)
10419	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
10420	else
10421	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
10422	#ifdef VBOX_WITH_STATISTICS
10423	switch (iCrReg)
10424	{
10425	case 0: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR0Read); break;
10426	case 2: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR2Read); break;
10427	case 3: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR3Read); break;
10428	case 4: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR4Read); break;
10429	case 8: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR8Read); break;
10430	}
10431	#endif
10432	Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
10433	return rcStrict;
10434	}
10435
10436
10437	/**
10438	* VM-exit helper for MOV to CRx (CRx write).
10439	*/
10440	static VBOXSTRICTRC vmxHCExitMovToCrX(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
10441	{
10442	HMVMX_CPUMCTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
10443
10444	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
10445	AssertMsg( rcStrict == VINF_SUCCESS
10446	\|\| rcStrict == VINF_IEM_RAISED_XCPT
10447	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10448
10449	switch (iCrReg)
10450	{
10451	case 0:
10452	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0
10453	\| HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
10454	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR0Write);
10455	Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
10456	break;
10457
10458	case 2:
10459	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR2Write);
10460	/* Nothing to do here, CR2 it's not part of the VMCS. */
10461	break;
10462
10463	case 3:
10464	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
10465	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR3Write);
10466	Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
10467	break;
10468
10469	case 4:
10470	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
10471	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR4Write);
10472	#ifdef IN_RING0
10473	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
10474	pVCpu->cpum.GstCtx.cr4, pVCpu->hmr0.s.fLoadSaveGuestXcr0));
10475	#else
10476	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr4));
10477	#endif
10478	break;
10479
10480	case 8:
10481	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged,
10482	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
10483	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR8Write);
10484	break;
10485
10486	default:
10487	AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
10488	break;
10489	}
10490
10491	if (rcStrict == VINF_IEM_RAISED_XCPT)
10492	{
10493	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
10494	rcStrict = VINF_SUCCESS;
10495	}
10496	return rcStrict;
10497	}
10498
10499
10500	/**
10501	* VM-exit exception handler for \#PF (Page-fault exception).
10502	*
10503	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
10504	*/
10505	static VBOXSTRICTRC vmxHCExitXcptPF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10506	{
10507	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10508	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
10509
10510	#ifdef IN_RING0
10511	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10512	if (!VM_IS_VMX_NESTED_PAGING(pVM))
10513	{ /* likely */ }
10514	else
10515	#endif
10516	{
10517	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF) && defined(IN_RING0)
10518	Assert(pVmxTransient->fIsNestedGuest \|\| pVCpu->hmr0.s.fUsingDebugLoop);
10519	#endif
10520	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
10521	if (!pVmxTransient->fVectoringDoublePF)
10522	{
10523	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
10524	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
10525	}
10526	else
10527	{
10528	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10529	Assert(!pVmxTransient->fIsNestedGuest);
10530	vmxHCSetPendingXcptDF(pVCpu);
10531	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
10532	}
10533	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestPF);
10534	return VINF_SUCCESS;
10535	}
10536
10537	Assert(!pVmxTransient->fIsNestedGuest);
10538
10539	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
10540	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
10541	if (pVmxTransient->fVectoringPF)
10542	{
10543	Assert(VCPU_2_VMXSTATE(pVCpu).Event.fPending);
10544	return VINF_EM_RAW_INJECT_TRPM_EVENT;
10545	}
10546
10547	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10548	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10549	AssertRCReturn(rc, rc);
10550
10551	Log4Func(("#PF: cs:rip=%#04x:%#RX64 err_code=%#RX32 exit_qual=%#RX64 cr3=%#RX64\n", pCtx->cs.Sel, pCtx->rip,
10552	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual, pCtx->cr3));
10553
10554	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
10555	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
10556
10557	Log4Func(("#PF: rc=%Rrc\n", rc));
10558	if (rc == VINF_SUCCESS)
10559	{
10560	/*
10561	* This is typically a shadow page table sync or a MMIO instruction. But we may have
10562	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
10563	*/
10564	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
10565	TRPMResetTrap(pVCpu);
10566	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitShadowPF);
10567	return rc;
10568	}
10569
10570	if (rc == VINF_EM_RAW_GUEST_TRAP)
10571	{
10572	if (!pVmxTransient->fVectoringDoublePF)
10573	{
10574	/* It's a guest page fault and needs to be reflected to the guest. */
10575	uint32_t const uGstErrorCode = TRPMGetErrorCode(pVCpu);
10576	TRPMResetTrap(pVCpu);
10577	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false; /* In case it's a contributory #PF. */
10578	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
10579	uGstErrorCode, pVmxTransient->uExitQual);
10580	}
10581	else
10582	{
10583	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10584	TRPMResetTrap(pVCpu);
10585	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
10586	vmxHCSetPendingXcptDF(pVCpu);
10587	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
10588	}
10589
10590	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestPF);
10591	return VINF_SUCCESS;
10592	}
10593
10594	TRPMResetTrap(pVCpu);
10595	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitShadowPFEM);
10596	return rc;
10597	}
10598
10599
10600	/**
10601	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
10602	*
10603	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
10604	*/
10605	static VBOXSTRICTRC vmxHCExitXcptMF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10606	{
10607	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10608	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestMF);
10609
10610	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
10611	AssertRCReturn(rc, rc);
10612
10613	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
10614	{
10615	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
10616	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
10617
10618	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
10619	* provides VM-exit instruction length. If this causes problem later,
10620	* disassemble the instruction like it's done on AMD-V. */
10621	int rc2 = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
10622	AssertRCReturn(rc2, rc2);
10623	return rc;
10624	}
10625
10626	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbExitInstr,
10627	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10628	return VINF_SUCCESS;
10629	}
10630
10631
10632	/**
10633	* VM-exit exception handler for \#BP (Breakpoint exception).
10634	*
10635	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
10636	*/
10637	static VBOXSTRICTRC vmxHCExitXcptBP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10638	{
10639	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10640	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestBP);
10641
10642	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10643	AssertRCReturn(rc, rc);
10644
10645	VBOXSTRICTRC rcStrict;
10646	if (!pVmxTransient->fIsNestedGuest)
10647	rcStrict = DBGFTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx));
10648	else
10649	rcStrict = VINF_EM_RAW_GUEST_TRAP;
10650
10651	if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
10652	{
10653	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10654	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10655	rcStrict = VINF_SUCCESS;
10656	}
10657
10658	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_DBG_BREAKPOINT);
10659	return rcStrict;
10660	}
10661
10662
10663	/**
10664	* VM-exit exception handler for \#AC (Alignment-check exception).
10665	*
10666	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
10667	*/
10668	static VBOXSTRICTRC vmxHCExitXcptAC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10669	{
10670	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10671
10672	/*
10673	* Detect #ACs caused by host having enabled split-lock detection.
10674	* Emulate such instructions.
10675	*/
10676	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo,
10677	CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS);
10678	AssertRCReturn(rc, rc);
10679	/** @todo detect split lock in cpu feature? */
10680	if ( /* 1. If 486-style alignment checks aren't enabled, then this must be a split-lock exception */
10681	!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM)
10682	/* 2. #AC cannot happen in rings 0-2 except for split-lock detection. */
10683	\|\| CPUMGetGuestCPL(pVCpu) != 3
10684	/* 3. When the EFLAGS.AC != 0 this can only be a split-lock case. */
10685	\|\| !(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_AC) )
10686	{
10687	/*
10688	* Check for debug/trace events and import state accordingly.
10689	*/
10690	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestACSplitLock);
10691	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10692	if ( !DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_VMX_SPLIT_LOCK)
10693	#ifdef IN_RING0
10694	&& !VBOXVMM_VMX_SPLIT_LOCK_ENABLED()
10695	#endif
10696	)
10697	{
10698	if (pVM->cCpus == 1)
10699	{
10700	#if 0 /** @todo r=bird: This is potentially wrong. Might have to just do a whole state sync above and mark everything changed to be safe... */
10701	rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
10702	#else
10703	rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10704	#endif
10705	AssertRCReturn(rc, rc);
10706	}
10707	}
10708	else
10709	{
10710	rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10711	AssertRCReturn(rc, rc);
10712
10713	VBOXVMM_XCPT_DF(pVCpu, &pVCpu->cpum.GstCtx);
10714
10715	if (DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_VMX_SPLIT_LOCK))
10716	{
10717	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, DBGFEVENT_VMX_SPLIT_LOCK, DBGFEVENTCTX_HM, 0);
10718	if (rcStrict != VINF_SUCCESS)
10719	return rcStrict;
10720	}
10721	}
10722
10723	/*
10724	* Emulate the instruction.
10725	*
10726	* We have to ignore the LOCK prefix here as we must not retrigger the
10727	* detection on the host. This isn't all that satisfactory, though...
10728	*/
10729	if (pVM->cCpus == 1)
10730	{
10731	Log8Func(("cs:rip=%#04x:%#RX64 rflags=%#RX64 cr0=%#RX64 split-lock #AC\n", pVCpu->cpum.GstCtx.cs.Sel,
10732	pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags, pVCpu->cpum.GstCtx.cr0));
10733
10734	/** @todo For SMP configs we should do a rendezvous here. */
10735	VBOXSTRICTRC rcStrict = IEMExecOneIgnoreLock(pVCpu);
10736	if (rcStrict == VINF_SUCCESS)
10737	#if 0 /** @todo r=bird: This is potentially wrong. Might have to just do a whole state sync above and mark everything changed to be safe... */
10738	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged,
10739	HM_CHANGED_GUEST_RIP
10740	\| HM_CHANGED_GUEST_RFLAGS
10741	\| HM_CHANGED_GUEST_GPRS_MASK
10742	\| HM_CHANGED_GUEST_CS
10743	\| HM_CHANGED_GUEST_SS);
10744	#else
10745	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
10746	#endif
10747	else if (rcStrict == VINF_IEM_RAISED_XCPT)
10748	{
10749	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
10750	rcStrict = VINF_SUCCESS;
10751	}
10752	return rcStrict;
10753	}
10754	Log8Func(("cs:rip=%#04x:%#RX64 rflags=%#RX64 cr0=%#RX64 split-lock #AC -> VINF_EM_EMULATE_SPLIT_LOCK\n",
10755	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags, pVCpu->cpum.GstCtx.cr0));
10756	return VINF_EM_EMULATE_SPLIT_LOCK;
10757	}
10758
10759	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestAC);
10760	Log8Func(("cs:rip=%#04x:%#RX64 rflags=%#RX64 cr0=%#RX64 cpl=%d -> #AC\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
10761	pVCpu->cpum.GstCtx.rflags, pVCpu->cpum.GstCtx.cr0, CPUMGetGuestCPL(pVCpu) ));
10762
10763	/* Re-inject it. We'll detect any nesting before getting here. */
10764	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10765	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10766	return VINF_SUCCESS;
10767	}
10768
10769
10770	/**
10771	* VM-exit exception handler for \#DB (Debug exception).
10772	*
10773	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
10774	*/
10775	static VBOXSTRICTRC vmxHCExitXcptDB(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10776	{
10777	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10778	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDB);
10779
10780	/*
10781	* Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
10782	*/
10783	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
10784
10785	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10786	uint64_t const uDR6 = X86_DR6_INIT_VAL
10787	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
10788	\| X86_DR6_BD \| X86_DR6_BS));
10789
10790	int rc;
10791	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10792	if (!pVmxTransient->fIsNestedGuest)
10793	{
10794	rc = DBGFTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, VCPU_2_VMXSTATE(pVCpu).fSingleInstruction);
10795
10796	/*
10797	* Prevents stepping twice over the same instruction when the guest is stepping using
10798	* EFLAGS.TF and the hypervisor debugger is stepping using MTF.
10799	* Testcase: DOSQEMM, break (using "ba x 1") at cs:rip 0x70:0x774 and step (using "t").
10800	*/
10801	if ( rc == VINF_EM_DBG_STEPPED
10802	&& (pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG))
10803	{
10804	Assert(VCPU_2_VMXSTATE(pVCpu).fSingleInstruction);
10805	rc = VINF_EM_RAW_GUEST_TRAP;
10806	}
10807	}
10808	else
10809	rc = VINF_EM_RAW_GUEST_TRAP;
10810	Log6Func(("rc=%Rrc\n", rc));
10811	if (rc == VINF_EM_RAW_GUEST_TRAP)
10812	{
10813	/*
10814	* The exception was for the guest. Update DR6, DR7.GD and
10815	* IA32_DEBUGCTL.LBR before forwarding it.
10816	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
10817	*/
10818	#ifdef IN_RING0
10819	VMMRZCallRing3Disable(pVCpu);
10820	HM_DISABLE_PREEMPT(pVCpu);
10821
10822	pCtx->dr[6] &= ~X86_DR6_B_MASK;
10823	pCtx->dr[6] \|= uDR6;
10824	if (CPUMIsGuestDebugStateActive(pVCpu))
10825	ASMSetDR6(pCtx->dr[6]);
10826
10827	HM_RESTORE_PREEMPT();
10828	VMMRZCallRing3Enable(pVCpu);
10829	#else
10830	/** @todo */
10831	#endif
10832
10833	rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
10834	AssertRCReturn(rc, rc);
10835
10836	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10837	pCtx->dr[7] &= ~(uint64_t)X86_DR7_GD;
10838
10839	/* Paranoia. */
10840	pCtx->dr[7] &= ~(uint64_t)X86_DR7_RAZ_MASK;
10841	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
10842
10843	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
10844	AssertRC(rc);
10845
10846	/*
10847	* Raise #DB in the guest.
10848	*
10849	* It is important to reflect exactly what the VM-exit gave us (preserving the
10850	* interruption-type) rather than use vmxHCSetPendingXcptDB() as the #DB could've
10851	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
10852	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
10853	*
10854	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
10855	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
10856	*/
10857	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10858	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10859	return VINF_SUCCESS;
10860	}
10861
10862	/*
10863	* Not a guest trap, must be a hypervisor related debug event then.
10864	* Update DR6 in case someone is interested in it.
10865	*/
10866	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10867	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
10868	CPUMSetHyperDR6(pVCpu, uDR6);
10869
10870	return rc;
10871	}
10872
10873
10874	/**
10875	* Hacks its way around the lovely mesa driver's backdoor accesses.
10876	*
10877	* @sa hmR0SvmHandleMesaDrvGp.
10878	*/
10879	static int vmxHCHandleMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
10880	{
10881	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
10882	RT_NOREF(pCtx);
10883
10884	/* For now we'll just skip the instruction. */
10885	return vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
10886	}
10887
10888
10889	/**
10890	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
10891	* backdoor logging w/o checking what it is running inside.
10892	*
10893	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
10894	* backdoor port and magic numbers loaded in registers.
10895	*
10896	* @returns true if it is, false if it isn't.
10897	* @sa hmR0SvmIsMesaDrvGp.
10898	*/
10899	DECLINLINE(bool) vmxHCIsMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
10900	{
10901	/* 0xed: IN eAX,dx */
10902	uint8_t abInstr[1];
10903	if (pVmxTransient->cbExitInstr != sizeof(abInstr))
10904	return false;
10905
10906	/* Check that it is #GP(0). */
10907	if (pVmxTransient->uExitIntErrorCode != 0)
10908	return false;
10909
10910	/* Check magic and port. */
10911	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
10912	/Log(("vmxHCIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
10913	if (pCtx->rax != UINT32_C(0x564d5868))
10914	return false;
10915	if (pCtx->dx != UINT32_C(0x5658))
10916	return false;
10917
10918	/* Flat ring-3 CS. */
10919	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
10920	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
10921	/Log(("vmxHCIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
10922	if (pCtx->cs.Attr.n.u2Dpl != 3)
10923	return false;
10924	if (pCtx->cs.u64Base != 0)
10925	return false;
10926
10927	/* Check opcode. */
10928	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
10929	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
10930	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
10931	/Log(("vmxHCIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
10932	if (RT_FAILURE(rc))
10933	return false;
10934	if (abInstr[0] != 0xed)
10935	return false;
10936
10937	return true;
10938	}
10939
10940
10941	/**
10942	* VM-exit exception handler for \#GP (General-protection exception).
10943	*
10944	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
10945	*/
10946	static VBOXSTRICTRC vmxHCExitXcptGP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10947	{
10948	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10949	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestGP);
10950
10951	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10952	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10953	#ifdef IN_RING0
10954	PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
10955	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
10956	{ /* likely */ }
10957	else
10958	#endif
10959	{
10960	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10961	# ifdef IN_RING0
10962	Assert(pVCpu->hmr0.s.fUsingDebugLoop \|\| VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
10963	# else
10964	Assert(/pVCpu->hmr0.s.fUsingDebugLoop \|\|/ VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
10965	# endif
10966	#endif
10967	/*
10968	* If the guest is not in real-mode or we have unrestricted guest execution support, or if we are
10969	* executing a nested-guest, reflect #GP to the guest or nested-guest.
10970	*/
10971	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10972	AssertRCReturn(rc, rc);
10973	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
10974	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
10975
10976	if ( pVmxTransient->fIsNestedGuest
10977	\|\| !VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv
10978	\|\| !vmxHCIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
10979	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10980	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10981	else
10982	rc = vmxHCHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
10983	return rc;
10984	}
10985
10986	#ifdef IN_RING0
10987	Assert(CPUMIsGuestInRealModeEx(pCtx));
10988	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fUnrestrictedGuest);
10989	Assert(!pVmxTransient->fIsNestedGuest);
10990
10991	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10992	AssertRCReturn(rc, rc);
10993
10994	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
10995	if (rcStrict == VINF_SUCCESS)
10996	{
10997	if (!CPUMIsGuestInRealModeEx(pCtx))
10998	{
10999	/*
11000	* The guest is no longer in real-mode, check if we can continue executing the
11001	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
11002	*/
11003	pVmcsInfoShared->RealMode.fRealOnV86Active = false;
11004	if (HMCanExecuteVmxGuest(pVCpu->CTX_SUFF(pVM), pVCpu, pCtx))
11005	{
11006	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
11007	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
11008	}
11009	else
11010	{
11011	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
11012	rcStrict = VINF_EM_RESCHEDULE;
11013	}
11014	}
11015	else
11016	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
11017	}
11018	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11019	{
11020	rcStrict = VINF_SUCCESS;
11021	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11022	}
11023	return VBOXSTRICTRC_VAL(rcStrict);
11024	#endif
11025	}
11026
11027
11028	/**
11029	* VM-exit exception handler wrapper for all other exceptions that are not handled
11030	* by a specific handler.
11031	*
11032	* This simply re-injects the exception back into the VM without any special
11033	* processing.
11034	*
11035	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
11036	*/
11037	static VBOXSTRICTRC vmxHCExitXcptOthers(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11038	{
11039	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11040
11041	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11042	# ifdef IN_RING0
11043	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11044	AssertMsg(pVCpu->hmr0.s.fUsingDebugLoop \|\| pVmcsInfo->pShared->RealMode.fRealOnV86Active \|\| pVmxTransient->fIsNestedGuest,
11045	("uVector=%#x u32XcptBitmap=%#X32\n",
11046	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
11047	NOREF(pVmcsInfo);
11048	# endif
11049	#endif
11050
11051	/*
11052	* Re-inject the exception into the guest. This cannot be a double-fault condition which
11053	* would have been handled while checking exits due to event delivery.
11054	*/
11055	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11056
11057	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11058	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11059	AssertRCReturn(rc, rc);
11060	Log4Func(("Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11061	#endif
11062
11063	#ifdef VBOX_WITH_STATISTICS
11064	switch (uVector)
11065	{
11066	case X86_XCPT_DE: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDE); break;
11067	case X86_XCPT_DB: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDB); break;
11068	case X86_XCPT_BP: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestBP); break;
11069	case X86_XCPT_OF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestOF); break;
11070	case X86_XCPT_BR: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestBR); break;
11071	case X86_XCPT_UD: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestUD); break;
11072	case X86_XCPT_NM: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestOF); break;
11073	case X86_XCPT_DF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDF); break;
11074	case X86_XCPT_TS: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestTS); break;
11075	case X86_XCPT_NP: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestNP); break;
11076	case X86_XCPT_SS: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestSS); break;
11077	case X86_XCPT_GP: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestGP); break;
11078	case X86_XCPT_PF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestPF); break;
11079	case X86_XCPT_MF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestMF); break;
11080	case X86_XCPT_AC: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestAC); break;
11081	case X86_XCPT_XF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestXF); break;
11082	default:
11083	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestXcpUnk);
11084	break;
11085	}
11086	#endif
11087
11088	/* We should never call this function for a page-fault, we'd need to pass on the fault address below otherwise. */
11089	Assert(!VMX_EXIT_INT_INFO_IS_XCPT_PF(pVmxTransient->uExitIntInfo));
11090	NOREF(uVector);
11091
11092	/* Re-inject the original exception into the guest. */
11093	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11094	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11095	return VINF_SUCCESS;
11096	}
11097
11098
11099	/**
11100	* VM-exit exception handler for all exceptions (except NMIs!).
11101	*
11102	* @remarks This may be called for both guests and nested-guests. Take care to not
11103	* make assumptions and avoid doing anything that is not relevant when
11104	* executing a nested-guest (e.g., Mesa driver hacks).
11105	*/
11106	static VBOXSTRICTRC vmxHCExitXcpt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11107	{
11108	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
11109
11110	/*
11111	* If this VM-exit occurred while delivering an event through the guest IDT, take
11112	* action based on the return code and additional hints (e.g. for page-faults)
11113	* that will be updated in the VMX transient structure.
11114	*/
11115	VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
11116	if (rcStrict == VINF_SUCCESS)
11117	{
11118	/*
11119	* If an exception caused a VM-exit due to delivery of an event, the original
11120	* event may have to be re-injected into the guest. We shall reinject it and
11121	* continue guest execution. However, page-fault is a complicated case and
11122	* needs additional processing done in vmxHCExitXcptPF().
11123	*/
11124	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11125	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11126	if ( !VCPU_2_VMXSTATE(pVCpu).Event.fPending
11127	\|\| uVector == X86_XCPT_PF)
11128	{
11129	switch (uVector)
11130	{
11131	case X86_XCPT_PF: return vmxHCExitXcptPF(pVCpu, pVmxTransient);
11132	case X86_XCPT_GP: return vmxHCExitXcptGP(pVCpu, pVmxTransient);
11133	case X86_XCPT_MF: return vmxHCExitXcptMF(pVCpu, pVmxTransient);
11134	case X86_XCPT_DB: return vmxHCExitXcptDB(pVCpu, pVmxTransient);
11135	case X86_XCPT_BP: return vmxHCExitXcptBP(pVCpu, pVmxTransient);
11136	case X86_XCPT_AC: return vmxHCExitXcptAC(pVCpu, pVmxTransient);
11137	default:
11138	return vmxHCExitXcptOthers(pVCpu, pVmxTransient);
11139	}
11140	}
11141	/* else: inject pending event before resuming guest execution. */
11142	}
11143	else if (rcStrict == VINF_HM_DOUBLE_FAULT)
11144	{
11145	Assert(VCPU_2_VMXSTATE(pVCpu).Event.fPending);
11146	rcStrict = VINF_SUCCESS;
11147	}
11148
11149	return rcStrict;
11150	}
11151	/** @} */
11152
11153
11154	/** @name VM-exit handlers.
11155	* @{
11156	*/
11157	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11158	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11159	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11160
11161	/**
11162	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11163	*/
11164	HMVMX_EXIT_DECL vmxHCExitExtInt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11165	{
11166	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11167	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitExtInt);
11168
11169	#ifdef IN_RING0
11170	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11171	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11172	return VINF_SUCCESS;
11173	return VINF_EM_RAW_INTERRUPT;
11174	#else
11175	return VINF_SUCCESS;
11176	#endif
11177	}
11178
11179
11180	/**
11181	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
11182	* VM-exit.
11183	*/
11184	HMVMX_EXIT_DECL vmxHCExitXcptOrNmi(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11185	{
11186	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11187	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitXcptNmi, y3);
11188
11189	vmxHCReadExitIntInfoVmcs(pVCpu, pVmxTransient);
11190
11191	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11192	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11193	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11194
11195	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11196	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
11197	&& uExitIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
11198	NOREF(pVmcsInfo);
11199
11200	VBOXSTRICTRC rcStrict;
11201	switch (uExitIntType)
11202	{
11203	#ifdef IN_RING0 /* NMIs should never reach R3. */
11204	/*
11205	* Host physical NMIs:
11206	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
11207	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
11208	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
11209	*
11210	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11211	* See Intel spec. 27.5.5 "Updating Non-Register State".
11212	*/
11213	case VMX_EXIT_INT_INFO_TYPE_NMI:
11214	{
11215	rcStrict = vmxHCExitHostNmi(pVCpu, pVmcsInfo);
11216	break;
11217	}
11218	#endif
11219
11220	/*
11221	* Privileged software exceptions (#DB from ICEBP),
11222	* Software exceptions (#BP and #OF),
11223	* Hardware exceptions:
11224	* Process the required exceptions and resume guest execution if possible.
11225	*/
11226	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11227	Assert(uVector == X86_XCPT_DB);
11228	RT_FALL_THRU();
11229	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11230	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uExitIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
11231	RT_FALL_THRU();
11232	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11233	{
11234	NOREF(uVector);
11235	vmxHCReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
11236	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
11237	vmxHCReadIdtVectoringInfoVmcs(pVCpu, pVmxTransient);
11238	vmxHCReadIdtVectoringErrorCodeVmcs(pVCpu, pVmxTransient);
11239
11240	rcStrict = vmxHCExitXcpt(pVCpu, pVmxTransient);
11241	break;
11242	}
11243
11244	default:
11245	{
11246	VCPU_2_VMXSTATE(pVCpu).u32HMError = pVmxTransient->uExitIntInfo;
11247	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11248	AssertMsgFailed(("Invalid/unexpected VM-exit interruption info %#x\n", pVmxTransient->uExitIntInfo));
11249	break;
11250	}
11251	}
11252
11253	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitXcptNmi, y3);
11254	return rcStrict;
11255	}
11256
11257
11258	/**
11259	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11260	*/
11261	HMVMX_EXIT_NSRC_DECL vmxHCExitIntWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11262	{
11263	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11264
11265	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11266	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11267	vmxHCClearIntWindowExitVmcs(pVCpu, pVmcsInfo);
11268
11269	/* Evaluate and deliver pending events and resume guest execution. */
11270	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitIntWindow);
11271	return VINF_SUCCESS;
11272	}
11273
11274
11275	/**
11276	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11277	*/
11278	HMVMX_EXIT_NSRC_DECL vmxHCExitNmiWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11279	{
11280	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11281
11282	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11283	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
11284	{
11285	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11286	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
11287	}
11288
11289	Assert(!CPUMIsGuestNmiBlocking(pVCpu));
11290
11291	/*
11292	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11293	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11294	*/
11295	uint32_t fIntrState;
11296	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11297	AssertRC(rc);
11298	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
11299	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
11300	{
11301	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11302	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11303
11304	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
11305	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, fIntrState);
11306	AssertRC(rc);
11307	}
11308
11309	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11310	vmxHCClearNmiWindowExitVmcs(pVCpu, pVmcsInfo);
11311
11312	/* Evaluate and deliver pending events and resume guest execution. */
11313	return VINF_SUCCESS;
11314	}
11315
11316
11317	/**
11318	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11319	*/
11320	HMVMX_EXIT_NSRC_DECL vmxHCExitWbinvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11321	{
11322	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11323	return vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
11324	}
11325
11326
11327	/**
11328	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11329	*/
11330	HMVMX_EXIT_NSRC_DECL vmxHCExitInvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11331	{
11332	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11333	return vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
11334	}
11335
11336
11337	/**
11338	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11339	*/
11340	HMVMX_EXIT_DECL vmxHCExitCpuid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11341	{
11342	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11343
11344	/*
11345	* Get the state we need and update the exit history entry.
11346	*/
11347	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11348	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
11349
11350	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
11351	AssertRCReturn(rc, rc);
11352
11353	VBOXSTRICTRC rcStrict;
11354	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11355	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
11356	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11357	if (!pExitRec)
11358	{
11359	/*
11360	* Regular CPUID instruction execution.
11361	*/
11362	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbExitInstr);
11363	if (rcStrict == VINF_SUCCESS)
11364	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11365	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11366	{
11367	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11368	rcStrict = VINF_SUCCESS;
11369	}
11370	}
11371	else
11372	{
11373	/*
11374	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
11375	*/
11376	int rc2 = vmxHCImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11377	AssertRCReturn(rc2, rc2);
11378
11379	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11380	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11381
11382	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11383	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
11384
11385	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11386	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11387	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11388	}
11389	return rcStrict;
11390	}
11391
11392
11393	/**
11394	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11395	*/
11396	HMVMX_EXIT_DECL vmxHCExitGetsec(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11397	{
11398	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11399
11400	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11401	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
11402	AssertRCReturn(rc, rc);
11403
11404	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
11405	return VINF_EM_RAW_EMULATE_INSTR;
11406
11407	AssertMsgFailed(("vmxHCExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
11408	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
11409	}
11410
11411
11412	/**
11413	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11414	*/
11415	HMVMX_EXIT_DECL vmxHCExitRdtsc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11416	{
11417	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11418
11419	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11420	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
11421	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
11422	AssertRCReturn(rc, rc);
11423
11424	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbExitInstr);
11425	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11426	{
11427	/* If we get a spurious VM-exit when TSC offsetting is enabled,
11428	we must reset offsetting on VM-entry. See @bugref{6634}. */
11429	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11430	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11431	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11432	}
11433	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11434	{
11435	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11436	rcStrict = VINF_SUCCESS;
11437	}
11438	return rcStrict;
11439	}
11440
11441
11442	/**
11443	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11444	*/
11445	HMVMX_EXIT_DECL vmxHCExitRdtscp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11446	{
11447	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11448
11449	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11450	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
11451	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
11452	AssertRCReturn(rc, rc);
11453
11454	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbExitInstr);
11455	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11456	{
11457	/* If we get a spurious VM-exit when TSC offsetting is enabled,
11458	we must reset offsetting on VM-reentry. See @bugref{6634}. */
11459	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11460	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11461	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11462	}
11463	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11464	{
11465	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11466	rcStrict = VINF_SUCCESS;
11467	}
11468	return rcStrict;
11469	}
11470
11471
11472	/**
11473	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11474	*/
11475	HMVMX_EXIT_DECL vmxHCExitRdpmc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11476	{
11477	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11478
11479	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11480	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
11481	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
11482	AssertRCReturn(rc, rc);
11483
11484	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11485	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
11486	if (RT_LIKELY(rc == VINF_SUCCESS))
11487	{
11488	rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
11489	Assert(pVmxTransient->cbExitInstr == 2);
11490	}
11491	else
11492	{
11493	AssertMsgFailed(("vmxHCExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11494	rc = VERR_EM_INTERPRETER;
11495	}
11496	return rc;
11497	}
11498
11499
11500	/**
11501	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11502	*/
11503	HMVMX_EXIT_DECL vmxHCExitVmcall(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11504	{
11505	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11506
11507	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11508	if (EMAreHypercallInstructionsEnabled(pVCpu))
11509	{
11510	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11511	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
11512	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
11513	AssertRCReturn(rc, rc);
11514
11515	/* Perform the hypercall. */
11516	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
11517	if (rcStrict == VINF_SUCCESS)
11518	{
11519	rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
11520	AssertRCReturn(rc, rc);
11521	}
11522	else
11523	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11524	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11525	\|\| RT_FAILURE(rcStrict));
11526
11527	/* If the hypercall changes anything other than guest's general-purpose registers,
11528	we would need to reload the guest changed bits here before VM-entry. */
11529	}
11530	else
11531	Log4Func(("Hypercalls not enabled\n"));
11532
11533	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11534	if (RT_FAILURE(rcStrict))
11535	{
11536	vmxHCSetPendingXcptUD(pVCpu);
11537	rcStrict = VINF_SUCCESS;
11538	}
11539
11540	return rcStrict;
11541	}
11542
11543
11544	/**
11545	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11546	*/
11547	HMVMX_EXIT_DECL vmxHCExitInvlpg(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11548	{
11549	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11550	#ifdef IN_RING0
11551	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging \|\| pVCpu->hmr0.s.fUsingDebugLoop);
11552	#endif
11553
11554	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11555	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
11556	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
11557	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
11558	AssertRCReturn(rc, rc);
11559
11560	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->uExitQual);
11561
11562	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
11563	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11564	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11565	{
11566	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11567	rcStrict = VINF_SUCCESS;
11568	}
11569	else
11570	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
11571	VBOXSTRICTRC_VAL(rcStrict)));
11572	return rcStrict;
11573	}
11574
11575
11576	/**
11577	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11578	*/
11579	HMVMX_EXIT_DECL vmxHCExitMonitor(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11580	{
11581	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11582
11583	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11584	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
11585	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
11586	AssertRCReturn(rc, rc);
11587
11588	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbExitInstr);
11589	if (rcStrict == VINF_SUCCESS)
11590	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11591	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11592	{
11593	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11594	rcStrict = VINF_SUCCESS;
11595	}
11596
11597	return rcStrict;
11598	}
11599
11600
11601	/**
11602	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11603	*/
11604	HMVMX_EXIT_DECL vmxHCExitMwait(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11605	{
11606	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11607
11608	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11609	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
11610	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
11611	AssertRCReturn(rc, rc);
11612
11613	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbExitInstr);
11614	if (RT_SUCCESS(rcStrict))
11615	{
11616	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11617	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
11618	rcStrict = VINF_SUCCESS;
11619	}
11620
11621	return rcStrict;
11622	}
11623
11624
11625	/**
11626	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11627	* VM-exit.
11628	*/
11629	HMVMX_EXIT_DECL vmxHCExitTripleFault(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11630	{
11631	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11632	return VINF_EM_RESET;
11633	}
11634
11635
11636	/**
11637	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11638	*/
11639	HMVMX_EXIT_DECL vmxHCExitHlt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11640	{
11641	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11642
11643	int rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
11644	AssertRCReturn(rc, rc);
11645
11646	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
11647	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
11648	rc = VINF_SUCCESS;
11649	else
11650	rc = VINF_EM_HALT;
11651
11652	if (rc != VINF_SUCCESS)
11653	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchHltToR3);
11654	return rc;
11655	}
11656
11657
11658	/**
11659	* VM-exit handler for instructions that result in a \#UD exception delivered to
11660	* the guest.
11661	*/
11662	HMVMX_EXIT_NSRC_DECL vmxHCExitSetPendingXcptUD(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11663	{
11664	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11665	vmxHCSetPendingXcptUD(pVCpu);
11666	return VINF_SUCCESS;
11667	}
11668
11669
11670	/**
11671	* VM-exit handler for expiry of the VMX-preemption timer.
11672	*/
11673	HMVMX_EXIT_DECL vmxHCExitPreemptTimer(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11674	{
11675	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11676
11677	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
11678	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11679	Log12(("vmxHCExitPreemptTimer:\n"));
11680
11681	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11682	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
11683	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11684	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitPreemptTimer);
11685	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11686	}
11687
11688
11689	/**
11690	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11691	*/
11692	HMVMX_EXIT_DECL vmxHCExitXsetbv(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11693	{
11694	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11695
11696	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11697	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
11698	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
11699	AssertRCReturn(rc, rc);
11700
11701	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbExitInstr);
11702	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
11703	: HM_CHANGED_RAISED_XCPT_MASK);
11704
11705	#ifdef IN_RING0
11706	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11707	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
11708	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
11709	{
11710	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
11711	vmxHCUpdateStartVmFunction(pVCpu);
11712	}
11713	#endif
11714
11715	return rcStrict;
11716	}
11717
11718
11719	/**
11720	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11721	*/
11722	HMVMX_EXIT_DECL vmxHCExitInvpcid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11723	{
11724	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11725
11726	/** @todo Enable the new code after finding a reliably guest test-case. */
11727	#if 1
11728	return VERR_EM_INTERPRETER;
11729	#else
11730	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
11731	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
11732	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
11733	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
11734	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
11735	AssertRCReturn(rc, rc);
11736
11737	/* Paranoia. Ensure this has a memory operand. */
11738	Assert(!pVmxTransient->ExitInstrInfo.Inv.u1Cleared0);
11739
11740	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
11741	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
11742	uint64_t const uType = CPUMIsGuestIn64BitCode(pVCpu) ? pVCpu->cpum.GstCtx.aGRegs[iGReg].u64
11743	: pVCpu->cpum.GstCtx.aGRegs[iGReg].u32;
11744
11745	RTGCPTR GCPtrDesc;
11746	HMVMX_DECODE_MEM_OPERAND(pVCpu, pVmxTransient->ExitInstrInfo.u, pVmxTransient->uExitQual, VMXMEMACCESS_READ, &GCPtrDesc);
11747
11748	VBOXSTRICTRC rcStrict = IEMExecDecodedInvpcid(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->ExitInstrInfo.Inv.iSegReg,
11749	GCPtrDesc, uType);
11750	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11751	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11752	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11753	{
11754	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11755	rcStrict = VINF_SUCCESS;
11756	}
11757	return rcStrict;
11758	#endif
11759	}
11760
11761
11762	/**
11763	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
11764	* VM-exit.
11765	*/
11766	HMVMX_EXIT_NSRC_DECL vmxHCExitErrInvalidGuestState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11767	{
11768	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11769	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11770	AssertRCReturn(rc, rc);
11771
11772	rc = vmxHCCheckCachedVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
11773	if (RT_FAILURE(rc))
11774	return rc;
11775
11776	uint32_t const uInvalidReason = vmxHCCheckGuestState(pVCpu, pVmcsInfo);
11777	NOREF(uInvalidReason);
11778
11779	#ifdef VBOX_STRICT
11780	uint32_t fIntrState;
11781	uint64_t u64Val;
11782	vmxHCReadEntryIntInfoVmcs(pVCpu, pVmxTransient);
11783	vmxHCReadEntryXcptErrorCodeVmcs(pVCpu, pVmxTransient);
11784	vmxHCReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
11785
11786	Log4(("uInvalidReason %u\n", uInvalidReason));
11787	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11788	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11789	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11790
11791	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &fIntrState); AssertRC(rc);
11792	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
11793	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
11794	Log4(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
11795	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_MASK, &u64Val); AssertRC(rc);
11796	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RX64\n", u64Val));
11797	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Val); AssertRC(rc);
11798	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RX64\n", u64Val));
11799	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_MASK, &u64Val); AssertRC(rc);
11800	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RX64\n", u64Val));
11801	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Val); AssertRC(rc);
11802	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RX64\n", u64Val));
11803	# ifdef IN_RING0
11804	if (pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
11805	{
11806	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11807	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11808	}
11809
11810	hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
11811	# endif
11812	#endif
11813
11814	return VERR_VMX_INVALID_GUEST_STATE;
11815	}
11816
11817	/**
11818	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
11819	*/
11820	HMVMX_EXIT_NSRC_DECL vmxHCExitErrUnexpected(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11821	{
11822	/*
11823	* Cumulative notes of all recognized but unexpected VM-exits.
11824	*
11825	* 1. This does -not- cover scenarios like a page-fault VM-exit occurring when
11826	* nested-paging is used.
11827	*
11828	* 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
11829	* emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
11830	* this function (and thereby stop VM execution) for handling such instructions.
11831	*
11832	*
11833	* VMX_EXIT_INIT_SIGNAL:
11834	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11835	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
11836	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
11837	*
11838	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
11839	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
11840	* See Intel spec. "23.8 Restrictions on VMX operation".
11841	*
11842	* VMX_EXIT_SIPI:
11843	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
11844	* activity state is used. We don't make use of it as our guests don't have direct
11845	* access to the host local APIC.
11846	*
11847	* See Intel spec. 25.3 "Other Causes of VM-exits".
11848	*
11849	* VMX_EXIT_IO_SMI:
11850	* VMX_EXIT_SMI:
11851	* This can only happen if we support dual-monitor treatment of SMI, which can be
11852	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
11853	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
11854	* VMX root mode or receive an SMI. If we get here, something funny is going on.
11855	*
11856	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11857	* See Intel spec. 25.3 "Other Causes of VM-Exits"
11858	*
11859	* VMX_EXIT_ERR_MSR_LOAD:
11860	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
11861	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
11862	* execution.
11863	*
11864	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
11865	*
11866	* VMX_EXIT_ERR_MACHINE_CHECK:
11867	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
11868	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
11869	* #MC exception abort class exception is raised. We thus cannot assume a
11870	* reasonable chance of continuing any sort of execution and we bail.
11871	*
11872	* See Intel spec. 15.1 "Machine-check Architecture".
11873	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
11874	*
11875	* VMX_EXIT_PML_FULL:
11876	* VMX_EXIT_VIRTUALIZED_EOI:
11877	* VMX_EXIT_APIC_WRITE:
11878	* We do not currently support any of these features and thus they are all unexpected
11879	* VM-exits.
11880	*
11881	* VMX_EXIT_GDTR_IDTR_ACCESS:
11882	* VMX_EXIT_LDTR_TR_ACCESS:
11883	* VMX_EXIT_RDRAND:
11884	* VMX_EXIT_RSM:
11885	* VMX_EXIT_VMFUNC:
11886	* VMX_EXIT_ENCLS:
11887	* VMX_EXIT_RDSEED:
11888	* VMX_EXIT_XSAVES:
11889	* VMX_EXIT_XRSTORS:
11890	* VMX_EXIT_UMWAIT:
11891	* VMX_EXIT_TPAUSE:
11892	* VMX_EXIT_LOADIWKEY:
11893	* These VM-exits are -not- caused unconditionally by execution of the corresponding
11894	* instruction. Any VM-exit for these instructions indicate a hardware problem,
11895	* unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
11896	*
11897	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
11898	*/
11899	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11900	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
11901	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
11902	}
11903
11904
11905	/**
11906	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11907	*/
11908	HMVMX_EXIT_DECL vmxHCExitRdmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11909	{
11910	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11911
11912	/** @todo Optimize this: We currently drag in the whole MSR state
11913	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11914	* MSRs required. That would require changes to IEM and possibly CPUM too.
11915	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11916	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11917	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
11918	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
11919	switch (idMsr)
11920	{
11921	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
11922	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
11923	}
11924
11925	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
11926	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, fImport);
11927	AssertRCReturn(rc, rc);
11928
11929	Log4Func(("ecx=%#RX32\n", idMsr));
11930
11931	#if defined(VBOX_STRICT) && defined(IN_RING0)
11932	Assert(!pVmxTransient->fIsNestedGuest);
11933	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
11934	{
11935	if ( vmxHCIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
11936	&& idMsr != MSR_K6_EFER)
11937	{
11938	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
11939	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
11940	}
11941	if (vmxHCIsLazyGuestMsr(pVCpu, idMsr))
11942	{
11943	Assert(pVmcsInfo->pvMsrBitmap);
11944	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
11945	if (fMsrpm & VMXMSRPM_ALLOW_RD)
11946	{
11947	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
11948	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
11949	}
11950	}
11951	}
11952	#endif
11953
11954	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbExitInstr);
11955	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitRdmsr);
11956	if (rcStrict == VINF_SUCCESS)
11957	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11958	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11959	{
11960	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11961	rcStrict = VINF_SUCCESS;
11962	}
11963	else
11964	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ \|\| rcStrict == VINF_EM_TRIPLE_FAULT,
11965	("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
11966
11967	return rcStrict;
11968	}
11969
11970
11971	/**
11972	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11973	*/
11974	HMVMX_EXIT_DECL vmxHCExitWrmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11975	{
11976	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11977
11978	/** @todo Optimize this: We currently drag in the whole MSR state
11979	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11980	* MSRs required. That would require changes to IEM and possibly CPUM too.
11981	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11982	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
11983	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
11984
11985	/*
11986	* The FS and GS base MSRs are not part of the above all-MSRs mask.
11987	* Although we don't need to fetch the base as it will be overwritten shortly, while
11988	* loading guest-state we would also load the entire segment register including limit
11989	* and attributes and thus we need to load them here.
11990	*/
11991	switch (idMsr)
11992	{
11993	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
11994	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
11995	}
11996
11997	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11998	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
11999	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, fImport);
12000	AssertRCReturn(rc, rc);
12001
12002	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
12003
12004	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbExitInstr);
12005	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitWrmsr);
12006
12007	if (rcStrict == VINF_SUCCESS)
12008	{
12009	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
12010
12011	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
12012	if ( idMsr == MSR_IA32_APICBASE
12013	\|\| ( idMsr >= MSR_IA32_X2APIC_START
12014	&& idMsr <= MSR_IA32_X2APIC_END))
12015	{
12016	/*
12017	* We've already saved the APIC related guest-state (TPR) in post-run phase.
12018	* When full APIC register virtualization is implemented we'll have to make
12019	* sure APIC state is saved from the VMCS before IEM changes it.
12020	*/
12021	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
12022	}
12023	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12024	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
12025	else if (idMsr == MSR_K6_EFER)
12026	{
12027	/*
12028	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
12029	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
12030	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
12031	*/
12032	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
12033	}
12034
12035	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not used. */
12036	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
12037	{
12038	switch (idMsr)
12039	{
12040	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12041	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12042	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12043	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_FS); break;
12044	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_GS); break;
12045	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
12046	default:
12047	{
12048	#ifdef IN_RING0
12049	if (vmxHCIsLazyGuestMsr(pVCpu, idMsr))
12050	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
12051	else if (vmxHCIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
12052	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12053	#else
12054	AssertMsgFailed(("TODO\n"));
12055	#endif
12056	break;
12057	}
12058	}
12059	}
12060	#if defined(VBOX_STRICT) && defined(IN_RING0)
12061	else
12062	{
12063	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12064	switch (idMsr)
12065	{
12066	case MSR_IA32_SYSENTER_CS:
12067	case MSR_IA32_SYSENTER_EIP:
12068	case MSR_IA32_SYSENTER_ESP:
12069	case MSR_K8_FS_BASE:
12070	case MSR_K8_GS_BASE:
12071	{
12072	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
12073	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
12074	}
12075
12076	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12077	default:
12078	{
12079	if (vmxHCIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
12080	{
12081	/* EFER MSR writes are always intercepted. */
12082	if (idMsr != MSR_K6_EFER)
12083	{
12084	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12085	idMsr));
12086	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
12087	}
12088	}
12089
12090	if (vmxHCIsLazyGuestMsr(pVCpu, idMsr))
12091	{
12092	Assert(pVmcsInfo->pvMsrBitmap);
12093	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
12094	if (fMsrpm & VMXMSRPM_ALLOW_WR)
12095	{
12096	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
12097	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
12098	}
12099	}
12100	break;
12101	}
12102	}
12103	}
12104	#endif /* VBOX_STRICT */
12105	}
12106	else if (rcStrict == VINF_IEM_RAISED_XCPT)
12107	{
12108	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12109	rcStrict = VINF_SUCCESS;
12110	}
12111	else
12112	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE \|\| rcStrict == VINF_EM_TRIPLE_FAULT,
12113	("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
12114
12115	return rcStrict;
12116	}
12117
12118
12119	/**
12120	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12121	*/
12122	HMVMX_EXIT_DECL vmxHCExitPause(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12123	{
12124	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12125
12126	/** @todo The guest has likely hit a contended spinlock. We might want to
12127	* poke a schedule different guest VCPU. */
12128	int rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
12129	if (RT_SUCCESS(rc))
12130	return VINF_EM_RAW_INTERRUPT;
12131
12132	AssertMsgFailed(("vmxHCExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
12133	return rc;
12134	}
12135
12136
12137	/**
12138	* VM-exit handler for when the TPR value is lowered below the specified
12139	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12140	*/
12141	HMVMX_EXIT_NSRC_DECL vmxHCExitTprBelowThreshold(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12142	{
12143	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12144	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
12145
12146	/*
12147	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
12148	* We'll re-evaluate pending interrupts and inject them before the next VM
12149	* entry so we can just continue execution here.
12150	*/
12151	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitTprBelowThreshold);
12152	return VINF_SUCCESS;
12153	}
12154
12155
12156	/**
12157	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12158	* VM-exit.
12159	*
12160	* @retval VINF_SUCCESS when guest execution can continue.
12161	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12162	* @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
12163	* incompatible guest state for VMX execution (real-on-v86 case).
12164	*/
12165	HMVMX_EXIT_DECL vmxHCExitMovCRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12166	{
12167	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12168	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitMovCRx, y2);
12169
12170	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12171	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
12172	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
12173
12174	VBOXSTRICTRC rcStrict;
12175	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
12176	uint64_t const uExitQual = pVmxTransient->uExitQual;
12177	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
12178	switch (uAccessType)
12179	{
12180	/*
12181	* MOV to CRx.
12182	*/
12183	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
12184	{
12185	/*
12186	* When PAE paging is used, the CPU will reload PAE PDPTEs from CR3 when the guest
12187	* changes certain bits even in CR0, CR4 (and not just CR3). We are currently fine
12188	* since IEM_CPUMCTX_EXTRN_MUST_MASK (used below) includes CR3 which will import
12189	* PAE PDPTEs as well.
12190	*/
12191	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
12192	AssertRCReturn(rc, rc);
12193
12194	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
12195	#ifdef IN_RING0
12196	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
12197	#endif
12198	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
12199	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
12200
12201	/*
12202	* MOV to CR3 only cause a VM-exit when one or more of the following are true:
12203	* - When nested paging isn't used.
12204	* - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
12205	* - We are executing in the VM debug loop.
12206	*/
12207	#ifdef IN_RING0
12208	Assert( iCrReg != 3
12209	\|\| !VM_IS_VMX_NESTED_PAGING(pVM)
12210	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
12211	\|\| pVCpu->hmr0.s.fUsingDebugLoop);
12212	#else
12213	Assert( iCrReg != 3
12214	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx));
12215	#endif
12216
12217	/* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
12218	Assert( iCrReg != 8
12219	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
12220
12221	rcStrict = vmxHCExitMovToCrX(pVCpu, pVmxTransient->cbExitInstr, iGReg, iCrReg);
12222	AssertMsg( rcStrict == VINF_SUCCESS
12223	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12224
12225	#ifdef IN_RING0
12226	/*
12227	* This is a kludge for handling switches back to real mode when we try to use
12228	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
12229	* deal with special selector values, so we have to return to ring-3 and run
12230	* there till the selector values are V86 mode compatible.
12231	*
12232	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
12233	* latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
12234	* this function.
12235	*/
12236	if ( iCrReg == 0
12237	&& rcStrict == VINF_SUCCESS
12238	&& !VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
12239	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
12240	&& (uOldCr0 & X86_CR0_PE)
12241	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
12242	{
12243	/** @todo Check selectors rather than returning all the time. */
12244	Assert(!pVmxTransient->fIsNestedGuest);
12245	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
12246	rcStrict = VINF_EM_RESCHEDULE_REM;
12247	}
12248	#endif
12249
12250	break;
12251	}
12252
12253	/*
12254	* MOV from CRx.
12255	*/
12256	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
12257	{
12258	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
12259	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
12260
12261	/*
12262	* MOV from CR3 only cause a VM-exit when one or more of the following are true:
12263	* - When nested paging isn't used.
12264	* - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
12265	* - We are executing in the VM debug loop.
12266	*/
12267	#ifdef IN_RING0
12268	Assert( iCrReg != 3
12269	\|\| !VM_IS_VMX_NESTED_PAGING(pVM)
12270	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
12271	\|\| pVCpu->hmr0.s.fLeaveDone);
12272	#else
12273	Assert( iCrReg != 3
12274	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx));
12275	#endif
12276
12277	/* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12278	Assert( iCrReg != 8
12279	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
12280
12281	rcStrict = vmxHCExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
12282	break;
12283	}
12284
12285	/*
12286	* CLTS (Clear Task-Switch Flag in CR0).
12287	*/
12288	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
12289	{
12290	rcStrict = vmxHCExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr);
12291	break;
12292	}
12293
12294	/*
12295	* LMSW (Load Machine-Status Word into CR0).
12296	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
12297	*/
12298	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
12299	{
12300	RTGCPTR GCPtrEffDst;
12301	uint8_t const cbInstr = pVmxTransient->cbExitInstr;
12302	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
12303	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
12304	if (fMemOperand)
12305	{
12306	vmxHCReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
12307	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
12308	}
12309	else
12310	GCPtrEffDst = NIL_RTGCPTR;
12311	rcStrict = vmxHCExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
12312	break;
12313	}
12314
12315	default:
12316	{
12317	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
12318	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
12319	}
12320	}
12321
12322	Assert((VCPU_2_VMXSTATE(pVCpu).fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
12323	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
12324	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
12325
12326	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitMovCRx, y2);
12327	NOREF(pVM);
12328	return rcStrict;
12329	}
12330
12331
12332	/**
12333	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12334	* VM-exit.
12335	*/
12336	HMVMX_EXIT_DECL vmxHCExitIoInstr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12337	{
12338	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12339	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitIO, y1);
12340
12341	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12342	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12343	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
12344	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
12345	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
12346	\| CPUMCTX_EXTRN_EFER);
12347	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12348	AssertRCReturn(rc, rc);
12349
12350	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12351	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
12352	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
12353	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12354	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
12355	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12356	bool const fDbgStepping = VCPU_2_VMXSTATE(pVCpu).fSingleInstruction;
12357	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
12358
12359	/*
12360	* Update exit history to see if this exit can be optimized.
12361	*/
12362	VBOXSTRICTRC rcStrict;
12363	PCEMEXITREC pExitRec = NULL;
12364	if ( !fGstStepping
12365	&& !fDbgStepping)
12366	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12367	!fIOString
12368	? !fIOWrite
12369	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12370	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12371	: !fIOWrite
12372	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12373	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12374	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12375	if (!pExitRec)
12376	{
12377	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
12378	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12379
12380	uint32_t const cbValue = s_aIOSizes[uIOSize];
12381	uint32_t const cbInstr = pVmxTransient->cbExitInstr;
12382	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12383	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
12384	if (fIOString)
12385	{
12386	/*
12387	* INS/OUTS - I/O String instruction.
12388	*
12389	* Use instruction-information if available, otherwise fall back on
12390	* interpreting the instruction.
12391	*/
12392	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12393	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
12394	bool const fInsOutsInfo = RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
12395	if (fInsOutsInfo)
12396	{
12397	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
12398	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12399	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12400	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12401	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
12402	if (fIOWrite)
12403	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12404	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
12405	else
12406	{
12407	/*
12408	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12409	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12410	* See Intel Instruction spec. for "INS".
12411	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12412	*/
12413	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
12414	}
12415	}
12416	else
12417	rcStrict = IEMExecOne(pVCpu);
12418
12419	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP);
12420	fUpdateRipAlready = true;
12421	}
12422	else
12423	{
12424	/*
12425	* IN/OUT - I/O instruction.
12426	*/
12427	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12428	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
12429	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
12430	if (fIOWrite)
12431	{
12432	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
12433	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitIOWrite);
12434	#ifdef IN_RING0
12435	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12436	&& !pCtx->eflags.Bits.u1TF)
12437	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
12438	#endif
12439	}
12440	else
12441	{
12442	uint32_t u32Result = 0;
12443	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12444	if (IOM_SUCCESS(rcStrict))
12445	{
12446	/* Save result of I/O IN instr. in AL/AX/EAX. */
12447	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
12448	}
12449	#ifdef IN_RING0
12450	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12451	&& !pCtx->eflags.Bits.u1TF)
12452	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
12453	#endif
12454	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitIORead);
12455	}
12456	}
12457
12458	if (IOM_SUCCESS(rcStrict))
12459	{
12460	if (!fUpdateRipAlready)
12461	{
12462	vmxHCAdvanceGuestRipBy(pVCpu, cbInstr);
12463	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP);
12464	}
12465
12466	/*
12467	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12468	* while booting Fedora 17 64-bit guest.
12469	*
12470	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12471	*/
12472	if (fIOString)
12473	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12474
12475	/*
12476	* If any I/O breakpoints are armed, we need to check if one triggered
12477	* and take appropriate action.
12478	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12479	*/
12480	rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
12481	AssertRCReturn(rc, rc);
12482
12483	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12484	* execution engines about whether hyper BPs and such are pending. */
12485	uint32_t const uDr7 = pCtx->dr[7];
12486	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12487	&& X86_DR7_ANY_RW_IO(uDr7)
12488	&& (pCtx->cr4 & X86_CR4_DE))
12489	\|\| DBGFBpIsHwIoArmed(pVM)))
12490	{
12491	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatDRxIoCheck);
12492
12493	#ifdef IN_RING0
12494	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12495	VMMRZCallRing3Disable(pVCpu);
12496	HM_DISABLE_PREEMPT(pVCpu);
12497
12498	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12499
12500	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
12501	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12502	{
12503	/* Raise #DB. */
12504	if (fIsGuestDbgActive)
12505	ASMSetDR6(pCtx->dr[6]);
12506	if (pCtx->dr[7] != uDr7)
12507	VCPU_2_VMXSTATE(pVCpu).fCtxChanged \|= HM_CHANGED_GUEST_DR7;
12508
12509	vmxHCSetPendingXcptDB(pVCpu);
12510	}
12511	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12512	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12513	else if ( rcStrict2 != VINF_SUCCESS
12514	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
12515	rcStrict = rcStrict2;
12516	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12517
12518	HM_RESTORE_PREEMPT();
12519	VMMRZCallRing3Enable(pVCpu);
12520	#else
12521	/** @todo */
12522	#endif
12523	}
12524	}
12525
12526	#ifdef VBOX_STRICT
12527	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12528	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
12529	Assert(!fIOWrite);
12530	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12531	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
12532	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
12533	Assert(fIOWrite);
12534	else
12535	{
12536	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12537	* statuses, that the VMM device and some others may return. See
12538	* IOM_SUCCESS() for guidance. */
12539	AssertMsg( RT_FAILURE(rcStrict)
12540	\|\| rcStrict == VINF_SUCCESS
12541	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
12542	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
12543	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
12544	\|\| rcStrict == VINF_EM_RAW_TO_R3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12545	# endif
12546	}
12547	#endif
12548	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitIO, y1);
12549	}
12550	else
12551	{
12552	/*
12553	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
12554	*/
12555	int rc2 = vmxHCImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12556	AssertRCReturn(rc2, rc2);
12557	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &VCPU_2_VMXSTATS(pVCpu).StatExitIOWrite : &VCPU_2_VMXSTATS(pVCpu).StatExitIORead
12558	: fIOWrite ? &VCPU_2_VMXSTATS(pVCpu).StatExitIOStringWrite : &VCPU_2_VMXSTATS(pVCpu).StatExitIOStringRead);
12559	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12560	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12561	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
12562	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
12563
12564	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12565	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
12566
12567	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12568	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12569	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12570	}
12571	return rcStrict;
12572	}
12573
12574
12575	/**
12576	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12577	* VM-exit.
12578	*/
12579	HMVMX_EXIT_DECL vmxHCExitTaskSwitch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12580	{
12581	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12582
12583	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
12584	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
12585	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12586	{
12587	vmxHCReadIdtVectoringInfoVmcs(pVCpu, pVmxTransient);
12588	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12589	{
12590	uint32_t uErrCode;
12591	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
12592	{
12593	vmxHCReadIdtVectoringErrorCodeVmcs(pVCpu, pVmxTransient);
12594	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12595	}
12596	else
12597	uErrCode = 0;
12598
12599	RTGCUINTPTR GCPtrFaultAddress;
12600	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(pVmxTransient->uIdtVectoringInfo))
12601	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
12602	else
12603	GCPtrFaultAddress = 0;
12604
12605	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
12606
12607	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12608	pVmxTransient->cbExitInstr, uErrCode, GCPtrFaultAddress);
12609
12610	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo),
12611	VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo)));
12612	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitTaskSwitch);
12613	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12614	}
12615	}
12616
12617	/* Fall back to the interpreter to emulate the task-switch. */
12618	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitTaskSwitch);
12619	return VERR_EM_INTERPRETER;
12620	}
12621
12622
12623	/**
12624	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12625	*/
12626	HMVMX_EXIT_DECL vmxHCExitMtf(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12627	{
12628	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12629
12630	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12631	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
12632	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
12633	AssertRC(rc);
12634	return VINF_EM_DBG_STEPPED;
12635	}
12636
12637
12638	/**
12639	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12640	*/
12641	HMVMX_EXIT_DECL vmxHCExitApicAccess(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12642	{
12643	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12644	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitApicAccess);
12645
12646	vmxHCReadExitIntInfoVmcs(pVCpu, pVmxTransient);
12647	vmxHCReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
12648	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
12649	vmxHCReadIdtVectoringInfoVmcs(pVCpu, pVmxTransient);
12650	vmxHCReadIdtVectoringErrorCodeVmcs(pVCpu, pVmxTransient);
12651
12652	/*
12653	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
12654	*/
12655	VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12656	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
12657	{
12658	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12659	if (RT_UNLIKELY(VCPU_2_VMXSTATE(pVCpu).Event.fPending))
12660	{
12661	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectInterpret);
12662	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12663	}
12664	}
12665	else
12666	{
12667	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
12668	return rcStrict;
12669	}
12670
12671	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12672	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12673	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
12674	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
12675	AssertRCReturn(rc, rc);
12676
12677	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12678	uint32_t const uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
12679	switch (uAccessType)
12680	{
12681	#ifdef IN_RING0
12682	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12683	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12684	{
12685	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
12686	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
12687	("vmxHCExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12688
12689	RTGCPHYS GCPhys = VCPU_2_VMXSTATE(pVCpu).vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
12690	GCPhys &= PAGE_BASE_GC_MASK;
12691	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
12692	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12693	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
12694
12695	rcStrict = IOMR0MmioPhysHandler(pVCpu->CTX_SUFF(pVM), pVCpu,
12696	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW, GCPhys);
12697	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12698	if ( rcStrict == VINF_SUCCESS
12699	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12700	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
12701	{
12702	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
12703	\| HM_CHANGED_GUEST_APIC_TPR);
12704	rcStrict = VINF_SUCCESS;
12705	}
12706	break;
12707	}
12708	#else
12709	/** @todo */
12710	#endif
12711
12712	default:
12713	{
12714	Log4Func(("uAccessType=%#x\n", uAccessType));
12715	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12716	break;
12717	}
12718	}
12719
12720	if (rcStrict != VINF_SUCCESS)
12721	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchApicAccessToR3);
12722	return rcStrict;
12723	}
12724
12725
12726	/**
12727	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12728	* VM-exit.
12729	*/
12730	HMVMX_EXIT_DECL vmxHCExitMovDRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12731	{
12732	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12733	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12734
12735	/*
12736	* We might also get this VM-exit if the nested-guest isn't intercepting MOV DRx accesses.
12737	* In such a case, rather than disabling MOV DRx intercepts and resuming execution, we
12738	* must emulate the MOV DRx access.
12739	*/
12740	if (!pVmxTransient->fIsNestedGuest)
12741	{
12742	/* We should -not- get this VM-exit if the guest's debug registers were active. */
12743	if (pVmxTransient->fWasGuestDebugStateActive)
12744	{
12745	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
12746	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
12747	}
12748
12749	if ( !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction
12750	&& !pVmxTransient->fWasHyperDebugStateActive)
12751	{
12752	Assert(!DBGFIsStepping(pVCpu));
12753	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
12754
12755	/* Don't intercept MOV DRx any more. */
12756	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
12757	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
12758	AssertRC(rc);
12759
12760	#ifdef IN_RING0
12761	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12762	VMMRZCallRing3Disable(pVCpu);
12763	HM_DISABLE_PREEMPT(pVCpu);
12764
12765	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12766	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12767	Assert(CPUMIsGuestDebugStateActive(pVCpu));
12768
12769	HM_RESTORE_PREEMPT();
12770	VMMRZCallRing3Enable(pVCpu);
12771	#else
12772	/** @todo */
12773	#endif
12774
12775	#ifdef VBOX_WITH_STATISTICS
12776	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
12777	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12778	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxWrite);
12779	else
12780	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxRead);
12781	#endif
12782	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatDRxContextSwitch);
12783	return VINF_SUCCESS;
12784	}
12785	}
12786
12787	/*
12788	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
12789	* The EFER MSR is always up-to-date.
12790	* Update the segment registers and DR7 from the CPU.
12791	*/
12792	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12793	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
12794	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
12795	AssertRCReturn(rc, rc);
12796	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
12797
12798	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
12799	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12800	{
12801	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12802	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
12803	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
12804	if (RT_SUCCESS(rc))
12805	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_DR7);
12806	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxWrite);
12807	}
12808	else
12809	{
12810	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12811	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
12812	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
12813	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxRead);
12814	}
12815
12816	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
12817	if (RT_SUCCESS(rc))
12818	{
12819	int rc2 = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
12820	AssertRCReturn(rc2, rc2);
12821	return VINF_SUCCESS;
12822	}
12823	return rc;
12824	}
12825
12826
12827	/**
12828	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12829	* Conditional VM-exit.
12830	*/
12831	HMVMX_EXIT_DECL vmxHCExitEptMisconfig(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12832	{
12833	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12834
12835	#ifdef IN_RING0
12836	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
12837
12838	vmxHCReadExitIntInfoVmcs(pVCpu, pVmxTransient);
12839	vmxHCReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
12840	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
12841	vmxHCReadIdtVectoringInfoVmcs(pVCpu, pVmxTransient);
12842	vmxHCReadIdtVectoringErrorCodeVmcs(pVCpu, pVmxTransient);
12843
12844	/*
12845	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
12846	*/
12847	VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12848	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
12849	{
12850	/*
12851	* In the unlikely case where delivering an event causes an EPT misconfig (MMIO), go back to
12852	* instruction emulation to inject the original event. Otherwise, injecting the original event
12853	* using hardware-assisted VMX would trigger the same EPT misconfig VM-exit again.
12854	*/
12855	if (!VCPU_2_VMXSTATE(pVCpu).Event.fPending)
12856	{ /* likely */ }
12857	else
12858	{
12859	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectInterpret);
12860	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12861	/** @todo NSTVMX: Think about how this should be handled. */
12862	if (pVmxTransient->fIsNestedGuest)
12863	return VERR_VMX_IPE_3;
12864	#endif
12865	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12866	}
12867	}
12868	else
12869	{
12870	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
12871	return rcStrict;
12872	}
12873
12874	/*
12875	* Get sufficient state and update the exit history entry.
12876	*/
12877	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12878	vmxHCReadGuestPhysicalAddrVmcs(pVCpu, pVmxTransient);
12879	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
12880	AssertRCReturn(rc, rc);
12881
12882	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
12883	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12884	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
12885	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12886	if (!pExitRec)
12887	{
12888	/*
12889	* If we succeed, resume guest execution.
12890	* If we fail in interpreting the instruction because we couldn't get the guest physical address
12891	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12892	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12893	* weird case. See @bugref{6043}.
12894	*/
12895	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
12896	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12897	/** @todo bird: We can probably just go straight to IOM here and assume that
12898	* it's MMIO, then fall back on PGM if that hunch didn't work out so
12899	* well. However, we need to address that aliasing workarounds that
12900	* PGMR0Trap0eHandlerNPMisconfig implements. So, some care is needed.
12901	*
12902	* Might also be interesting to see if we can get this done more or
12903	* less locklessly inside IOM. Need to consider the lookup table
12904	* updating and use a bit more carefully first (or do all updates via
12905	* rendezvous) */
12906	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
12907	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12908	if ( rcStrict == VINF_SUCCESS
12909	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12910	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
12911	{
12912	/* Successfully handled MMIO operation. */
12913	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
12914	\| HM_CHANGED_GUEST_APIC_TPR);
12915	rcStrict = VINF_SUCCESS;
12916	}
12917	}
12918	else
12919	{
12920	/*
12921	* Frequent exit or something needing probing. Call EMHistoryExec.
12922	*/
12923	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12924	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12925
12926	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12927	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
12928
12929	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12930	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12931	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12932	}
12933	return rcStrict;
12934	#else
12935	AssertFailed();
12936	return VERR_VMX_IPE_3; /* Should never happen with Apple HV in R3. */
12937	#endif
12938	}
12939
12940
12941	/**
12942	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12943	* VM-exit.
12944	*/
12945	HMVMX_EXIT_DECL vmxHCExitEptViolation(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12946	{
12947	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12948	#ifdef IN_RING0
12949	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
12950
12951	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
12952	vmxHCReadExitIntInfoVmcs(pVCpu, pVmxTransient);
12953	vmxHCReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
12954	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
12955	vmxHCReadIdtVectoringInfoVmcs(pVCpu, pVmxTransient);
12956	vmxHCReadIdtVectoringErrorCodeVmcs(pVCpu, pVmxTransient);
12957
12958	/*
12959	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
12960	*/
12961	VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12962	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
12963	{
12964	/*
12965	* If delivery of an event causes an EPT violation (true nested #PF and not MMIO),
12966	* we shall resolve the nested #PF and re-inject the original event.
12967	*/
12968	if (VCPU_2_VMXSTATE(pVCpu).Event.fPending)
12969	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectReflectNPF);
12970	}
12971	else
12972	{
12973	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
12974	return rcStrict;
12975	}
12976
12977	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12978	vmxHCReadGuestPhysicalAddrVmcs(pVCpu, pVmxTransient);
12979	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
12980	AssertRCReturn(rc, rc);
12981
12982	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
12983	uint64_t const uExitQual = pVmxTransient->uExitQual;
12984	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", uExitQual));
12985
12986	RTGCUINT uErrorCode = 0;
12987	if (uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_INSTR_FETCH)
12988	uErrorCode \|= X86_TRAP_PF_ID;
12989	if (uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE)
12990	uErrorCode \|= X86_TRAP_PF_RW;
12991	if (uExitQual & (VMX_EXIT_QUAL_EPT_ENTRY_READ \| VMX_EXIT_QUAL_EPT_ENTRY_WRITE \| VMX_EXIT_QUAL_EPT_ENTRY_EXECUTE))
12992	uErrorCode \|= X86_TRAP_PF_P;
12993
12994	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12995	Log4Func(("at %#RX64 (%#RX64 errcode=%#x) cs:rip=%#04x:%#RX64\n", GCPhys, uExitQual, uErrorCode, pCtx->cs.Sel, pCtx->rip));
12996
12997	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
12998
12999	/*
13000	* Handle the pagefault trap for the nested shadow table.
13001	*/
13002	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
13003	rcStrict = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
13004	TRPMResetTrap(pVCpu);
13005
13006	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
13007	if ( rcStrict == VINF_SUCCESS
13008	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
13009	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
13010	{
13011	/* Successfully synced our nested page tables. */
13012	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitReasonNpf);
13013	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
13014	return VINF_SUCCESS;
13015	}
13016	#else
13017	PVM pVM = pVCpu->CTX_SUFF(pVM);
13018	uint64_t const uHostTsc = ASMReadTSC(); RT_NOREF(uHostTsc);
13019	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13020	vmxHCReadGuestPhysicalAddrVmcs(pVCpu, pVmxTransient);
13021	vmxHCImportGuestRip(pVCpu);
13022	vmxHCImportGuestSegReg(pVCpu, X86_SREG_CS);
13023
13024	/*
13025	* Ask PGM for information about the given GCPhys. We need to check if we're
13026	* out of sync first.
13027	*/
13028	NEMHCDARWINHMACPCCSTATE State = { RT_BOOL(pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE), false, false };
13029	PGMPHYSNEMPAGEINFO Info;
13030	int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, pVmxTransient->uGuestPhysicalAddr, State.fWriteAccess, &Info,
13031	nemR3DarwinHandleMemoryAccessPageCheckerCallback, &State);
13032	if (RT_SUCCESS(rc))
13033	{
13034	if (Info.fNemProt & ( RT_BOOL(pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE)
13035	? NEM_PAGE_PROT_WRITE : NEM_PAGE_PROT_READ))
13036	{
13037	if (State.fCanResume)
13038	{
13039	Log4(("MemExit/%u: %04x:%08RX64: %RGp (=>%RHp) %s fProt=%u%s%s%s; restarting\n",
13040	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13041	pVmxTransient->uGuestPhysicalAddr, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
13042	Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
13043	State.fDidSomething ? "" : " no-change"));
13044	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_MEMORY_ACCESS),
13045	pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, uHostTsc);
13046	return VINF_SUCCESS;
13047	}
13048	}
13049
13050	Log4(("MemExit/%u: %04x:%08RX64: %RGp (=>%RHp) %s fProt=%u%s%s%s; emulating\n",
13051	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13052	pVmxTransient->uGuestPhysicalAddr, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
13053	Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
13054	State.fDidSomething ? "" : " no-change"));
13055	}
13056	else
13057	Log4(("MemExit/%u: %04x:%08RX64: %RGp rc=%Rrc%s; emulating\n",
13058	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13059	pVmxTransient->uGuestPhysicalAddr, rc, State.fDidSomething ? " modified-backing" : ""));
13060
13061	/*
13062	* Emulate the memory access, either access handler or special memory.
13063	*/
13064	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
13065	RT_BOOL(pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE)
13066	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
13067	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
13068	pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, uHostTsc);
13069
13070	rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13071	AssertRCReturn(rc, rc);
13072
13073	VBOXSTRICTRC rcStrict;
13074	if (!pExitRec)
13075	rcStrict = IEMExecOne(pVCpu);
13076	else
13077	{
13078	/* Frequent access or probing. */
13079	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
13080	Log4(("MemExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
13081	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13082	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
13083	}
13084
13085	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
13086	#endif
13087
13088	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13089	return rcStrict;
13090	}
13091
13092
13093	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13094	/**
13095	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
13096	*/
13097	HMVMX_EXIT_DECL vmxHCExitVmclear(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13098	{
13099	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13100
13101	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13102	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
13103	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13104	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
13105	\| CPUMCTX_EXTRN_HWVIRT
13106	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13107	AssertRCReturn(rc, rc);
13108
13109	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13110
13111	VMXVEXITINFO ExitInfo;
13112	RT_ZERO(ExitInfo);
13113	ExitInfo.uReason = pVmxTransient->uExitReason;
13114	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13115	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13116	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13117	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13118
13119	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
13120	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13121	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
13122	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13123	{
13124	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13125	rcStrict = VINF_SUCCESS;
13126	}
13127	return rcStrict;
13128	}
13129
13130
13131	/**
13132	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
13133	*/
13134	HMVMX_EXIT_DECL vmxHCExitVmlaunch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13135	{
13136	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13137
13138	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
13139	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
13140	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13141	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13142	AssertRCReturn(rc, rc);
13143
13144	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13145
13146	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
13147	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMLAUNCH);
13148	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
13149	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13150	{
13151	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
13152	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
13153	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
13154	}
13155	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
13156	return rcStrict;
13157	}
13158
13159
13160	/**
13161	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
13162	*/
13163	HMVMX_EXIT_DECL vmxHCExitVmptrld(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13164	{
13165	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13166
13167	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13168	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
13169	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13170	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
13171	\| CPUMCTX_EXTRN_HWVIRT
13172	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13173	AssertRCReturn(rc, rc);
13174
13175	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13176
13177	VMXVEXITINFO ExitInfo;
13178	RT_ZERO(ExitInfo);
13179	ExitInfo.uReason = pVmxTransient->uExitReason;
13180	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13181	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13182	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13183	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13184
13185	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
13186	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13187	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
13188	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13189	{
13190	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13191	rcStrict = VINF_SUCCESS;
13192	}
13193	return rcStrict;
13194	}
13195
13196
13197	/**
13198	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
13199	*/
13200	HMVMX_EXIT_DECL vmxHCExitVmptrst(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13201	{
13202	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13203
13204	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13205	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
13206	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13207	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
13208	\| CPUMCTX_EXTRN_HWVIRT
13209	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13210	AssertRCReturn(rc, rc);
13211
13212	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13213
13214	VMXVEXITINFO ExitInfo;
13215	RT_ZERO(ExitInfo);
13216	ExitInfo.uReason = pVmxTransient->uExitReason;
13217	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13218	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13219	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13220	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
13221
13222	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
13223	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13224	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13225	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13226	{
13227	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13228	rcStrict = VINF_SUCCESS;
13229	}
13230	return rcStrict;
13231	}
13232
13233
13234	/**
13235	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
13236	*/
13237	HMVMX_EXIT_DECL vmxHCExitVmread(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13238	{
13239	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13240
13241	/*
13242	* Strictly speaking we should not get VMREAD VM-exits for shadow VMCS fields and
13243	* thus might not need to import the shadow VMCS state, it's safer just in case
13244	* code elsewhere dares look at unsynced VMCS fields.
13245	*/
13246	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13247	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
13248	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13249	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
13250	\| CPUMCTX_EXTRN_HWVIRT
13251	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13252	AssertRCReturn(rc, rc);
13253
13254	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13255
13256	VMXVEXITINFO ExitInfo;
13257	RT_ZERO(ExitInfo);
13258	ExitInfo.uReason = pVmxTransient->uExitReason;
13259	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13260	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13261	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13262	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
13263	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
13264
13265	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
13266	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13267	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13268	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13269	{
13270	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13271	rcStrict = VINF_SUCCESS;
13272	}
13273	return rcStrict;
13274	}
13275
13276
13277	/**
13278	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
13279	*/
13280	HMVMX_EXIT_DECL vmxHCExitVmresume(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13281	{
13282	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13283
13284	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
13285	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
13286	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13287	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13288	AssertRCReturn(rc, rc);
13289
13290	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13291
13292	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
13293	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMRESUME);
13294	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
13295	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13296	{
13297	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
13298	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
13299	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
13300	}
13301	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
13302	return rcStrict;
13303	}
13304
13305
13306	/**
13307	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
13308	*/
13309	HMVMX_EXIT_DECL vmxHCExitVmwrite(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13310	{
13311	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13312
13313	/*
13314	* Although we should not get VMWRITE VM-exits for shadow VMCS fields, since our HM hook
13315	* gets invoked when IEM's VMWRITE instruction emulation modifies the current VMCS and it
13316	* flags re-loading the entire shadow VMCS, we should save the entire shadow VMCS here.
13317	*/
13318	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13319	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
13320	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13321	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
13322	\| CPUMCTX_EXTRN_HWVIRT
13323	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13324	AssertRCReturn(rc, rc);
13325
13326	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13327
13328	VMXVEXITINFO ExitInfo;
13329	RT_ZERO(ExitInfo);
13330	ExitInfo.uReason = pVmxTransient->uExitReason;
13331	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13332	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13333	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13334	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
13335	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13336
13337	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
13338	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13339	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
13340	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13341	{
13342	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13343	rcStrict = VINF_SUCCESS;
13344	}
13345	return rcStrict;
13346	}
13347
13348
13349	/**
13350	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
13351	*/
13352	HMVMX_EXIT_DECL vmxHCExitVmxoff(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13353	{
13354	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13355
13356	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13357	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
13358	\| CPUMCTX_EXTRN_HWVIRT
13359	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13360	AssertRCReturn(rc, rc);
13361
13362	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13363
13364	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbExitInstr);
13365	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13366	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
13367	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13368	{
13369	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13370	rcStrict = VINF_SUCCESS;
13371	}
13372	return rcStrict;
13373	}
13374
13375
13376	/**
13377	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
13378	*/
13379	HMVMX_EXIT_DECL vmxHCExitVmxon(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13380	{
13381	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13382
13383	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13384	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
13385	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13386	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
13387	\| CPUMCTX_EXTRN_HWVIRT
13388	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13389	AssertRCReturn(rc, rc);
13390
13391	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13392
13393	VMXVEXITINFO ExitInfo;
13394	RT_ZERO(ExitInfo);
13395	ExitInfo.uReason = pVmxTransient->uExitReason;
13396	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13397	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13398	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13399	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13400
13401	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
13402	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13403	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
13404	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13405	{
13406	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13407	rcStrict = VINF_SUCCESS;
13408	}
13409	return rcStrict;
13410	}
13411
13412
13413	/**
13414	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
13415	*/
13416	HMVMX_EXIT_DECL vmxHCExitInvvpid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13417	{
13418	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13419
13420	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13421	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
13422	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13423	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
13424	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13425	AssertRCReturn(rc, rc);
13426
13427	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13428
13429	VMXVEXITINFO ExitInfo;
13430	RT_ZERO(ExitInfo);
13431	ExitInfo.uReason = pVmxTransient->uExitReason;
13432	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13433	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13434	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13435	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13436
13437	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
13438	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13439	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13440	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13441	{
13442	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13443	rcStrict = VINF_SUCCESS;
13444	}
13445	return rcStrict;
13446	}
13447	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13448	/** @} */
13449
13450
13451	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13452	/** @name Nested-guest VM-exit handlers.
13453	* @{
13454	*/
13455	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13456	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13457	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13458
13459	/**
13460	* Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
13461	* Conditional VM-exit.
13462	*/
13463	HMVMX_EXIT_DECL vmxHCExitXcptOrNmiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13464	{
13465	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13466
13467	vmxHCReadExitIntInfoVmcs(pVCpu, pVmxTransient);
13468
13469	uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
13470	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
13471	Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));
13472
13473	switch (uExitIntType)
13474	{
13475	/*
13476	* Physical NMIs:
13477	* We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the host.
13478	*/
13479	case VMX_EXIT_INT_INFO_TYPE_NMI:
13480	return vmxHCExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
13481
13482	/*
13483	* Hardware exceptions,
13484	* Software exceptions,
13485	* Privileged software exceptions:
13486	* Figure out if the exception must be delivered to the guest or the nested-guest.
13487	*/
13488	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
13489	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
13490	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
13491	{
13492	vmxHCReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
13493	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13494	vmxHCReadIdtVectoringInfoVmcs(pVCpu, pVmxTransient);
13495	vmxHCReadIdtVectoringErrorCodeVmcs(pVCpu, pVmxTransient);
13496
13497	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13498	bool const fIntercept = CPUMIsGuestVmxXcptInterceptSet(pCtx, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo),
13499	pVmxTransient->uExitIntErrorCode);
13500	if (fIntercept)
13501	{
13502	/* Exit qualification is required for debug and page-fault exceptions. */
13503	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13504
13505	/*
13506	* For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
13507	* software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
13508	* length. However, if delivery of a software interrupt, software exception or privileged
13509	* software exception causes a VM-exit, that too provides the VM-exit instruction length.
13510	*/
13511	VMXVEXITINFO ExitInfo;
13512	RT_ZERO(ExitInfo);
13513	ExitInfo.uReason = pVmxTransient->uExitReason;
13514	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13515	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13516
13517	VMXVEXITEVENTINFO ExitEventInfo;
13518	RT_ZERO(ExitEventInfo);
13519	ExitEventInfo.uExitIntInfo = pVmxTransient->uExitIntInfo;
13520	ExitEventInfo.uExitIntErrCode = pVmxTransient->uExitIntErrorCode;
13521	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
13522	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
13523
13524	#ifdef DEBUG_ramshankar
13525	vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13526	Log4Func(("exit_int_info=%#RX32 err_code=%#RX32 exit_qual=%#RX64\n", pVmxTransient->uExitIntInfo,
13527	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual));
13528	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
13529	{
13530	Log4Func(("idt_info=%#RX32 idt_errcode=%#RX32 cr2=%#RX64\n", pVmxTransient->uIdtVectoringInfo,
13531	pVmxTransient->uIdtVectoringErrorCode, pCtx->cr2));
13532	}
13533	#endif
13534	return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
13535	}
13536
13537	/* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs in vmxHCExitXcptPF. */
13538	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
13539	return vmxHCExitXcpt(pVCpu, pVmxTransient);
13540	}
13541
13542	/*
13543	* Software interrupts:
13544	* VM-exits cannot be caused by software interrupts.
13545	*
13546	* External interrupts:
13547	* This should only happen when "acknowledge external interrupts on VM-exit"
13548	* control is set. However, we never set this when executing a guest or
13549	* nested-guest. For nested-guests it is emulated while injecting interrupts into
13550	* the guest.
13551	*/
13552	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
13553	case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
13554	default:
13555	{
13556	VCPU_2_VMXSTATE(pVCpu).u32HMError = pVmxTransient->uExitIntInfo;
13557	return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
13558	}
13559	}
13560	}
13561
13562
13563	/**
13564	* Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
13565	* Unconditional VM-exit.
13566	*/
13567	HMVMX_EXIT_DECL vmxHCExitTripleFaultNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13568	{
13569	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13570	return IEMExecVmxVmexitTripleFault(pVCpu);
13571	}
13572
13573
13574	/**
13575	* Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
13576	*/
13577	HMVMX_EXIT_NSRC_DECL vmxHCExitIntWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13578	{
13579	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13580
13581	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
13582	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
13583	return vmxHCExitIntWindow(pVCpu, pVmxTransient);
13584	}
13585
13586
13587	/**
13588	* Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
13589	*/
13590	HMVMX_EXIT_NSRC_DECL vmxHCExitNmiWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13591	{
13592	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13593
13594	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
13595	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
13596	return vmxHCExitIntWindow(pVCpu, pVmxTransient);
13597	}
13598
13599
13600	/**
13601	* Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
13602	* Unconditional VM-exit.
13603	*/
13604	HMVMX_EXIT_DECL vmxHCExitTaskSwitchNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13605	{
13606	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13607
13608	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13609	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13610	vmxHCReadIdtVectoringInfoVmcs(pVCpu, pVmxTransient);
13611	vmxHCReadIdtVectoringErrorCodeVmcs(pVCpu, pVmxTransient);
13612
13613	VMXVEXITINFO ExitInfo;
13614	RT_ZERO(ExitInfo);
13615	ExitInfo.uReason = pVmxTransient->uExitReason;
13616	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13617	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13618
13619	VMXVEXITEVENTINFO ExitEventInfo;
13620	RT_ZERO(ExitEventInfo);
13621	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
13622	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
13623	return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
13624	}
13625
13626
13627	/**
13628	* Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
13629	*/
13630	HMVMX_EXIT_DECL vmxHCExitHltNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13631	{
13632	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13633
13634	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
13635	{
13636	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13637	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
13638	}
13639	return vmxHCExitHlt(pVCpu, pVmxTransient);
13640	}
13641
13642
13643	/**
13644	* Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
13645	*/
13646	HMVMX_EXIT_DECL vmxHCExitInvlpgNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13647	{
13648	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13649
13650	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
13651	{
13652	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13653	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13654
13655	VMXVEXITINFO ExitInfo;
13656	RT_ZERO(ExitInfo);
13657	ExitInfo.uReason = pVmxTransient->uExitReason;
13658	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13659	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13660	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
13661	}
13662	return vmxHCExitInvlpg(pVCpu, pVmxTransient);
13663	}
13664
13665
13666	/**
13667	* Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
13668	*/
13669	HMVMX_EXIT_DECL vmxHCExitRdpmcNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13670	{
13671	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13672
13673	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
13674	{
13675	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13676	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
13677	}
13678	return vmxHCExitRdpmc(pVCpu, pVmxTransient);
13679	}
13680
13681
13682	/**
13683	* Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
13684	* (VMX_EXIT_VMWRITE). Conditional VM-exit.
13685	*/
13686	HMVMX_EXIT_DECL vmxHCExitVmreadVmwriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13687	{
13688	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13689
13690	Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD
13691	\|\| pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
13692
13693	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
13694
13695	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
13696	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
13697	uint64_t u64VmcsField = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
13698
13699	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
13700	if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
13701	u64VmcsField &= UINT64_C(0xffffffff);
13702
13703	if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64VmcsField))
13704	{
13705	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13706	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13707
13708	VMXVEXITINFO ExitInfo;
13709	RT_ZERO(ExitInfo);
13710	ExitInfo.uReason = pVmxTransient->uExitReason;
13711	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13712	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13713	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
13714	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
13715	}
13716
13717	if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
13718	return vmxHCExitVmread(pVCpu, pVmxTransient);
13719	return vmxHCExitVmwrite(pVCpu, pVmxTransient);
13720	}
13721
13722
13723	/**
13724	* Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
13725	*/
13726	HMVMX_EXIT_DECL vmxHCExitRdtscNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13727	{
13728	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13729
13730	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
13731	{
13732	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13733	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
13734	}
13735
13736	return vmxHCExitRdtsc(pVCpu, pVmxTransient);
13737	}
13738
13739
13740	/**
13741	* Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
13742	* Conditional VM-exit.
13743	*/
13744	HMVMX_EXIT_DECL vmxHCExitMovCRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13745	{
13746	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13747
13748	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13749	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13750
13751	VBOXSTRICTRC rcStrict;
13752	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
13753	switch (uAccessType)
13754	{
13755	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
13756	{
13757	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
13758	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
13759	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
13760	uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
13761
13762	bool fIntercept;
13763	switch (iCrReg)
13764	{
13765	case 0:
13766	case 4:
13767	fIntercept = CPUMIsGuestVmxMovToCr0Cr4InterceptSet(&pVCpu->cpum.GstCtx, iCrReg, uNewCrX);
13768	break;
13769
13770	case 3:
13771	fIntercept = CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCrX);
13772	break;
13773
13774	case 8:
13775	fIntercept = CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_CR8_LOAD_EXIT);
13776	break;
13777
13778	default:
13779	fIntercept = false;
13780	break;
13781	}
13782	if (fIntercept)
13783	{
13784	VMXVEXITINFO ExitInfo;
13785	RT_ZERO(ExitInfo);
13786	ExitInfo.uReason = pVmxTransient->uExitReason;
13787	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13788	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13789	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
13790	}
13791	else
13792	{
13793	int const rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13794	AssertRCReturn(rc, rc);
13795	rcStrict = vmxHCExitMovToCrX(pVCpu, pVmxTransient->cbExitInstr, iGReg, iCrReg);
13796	}
13797	break;
13798	}
13799
13800	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
13801	{
13802	/*
13803	* CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
13804	* CR2 reads do not cause a VM-exit.
13805	* CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
13806	* CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
13807	*/
13808	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
13809	if ( iCrReg == 3
13810	\|\| iCrReg == 8)
13811	{
13812	static const uint32_t s_auCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
13813	0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
13814	uint32_t const uIntercept = s_auCrXReadIntercepts[iCrReg];
13815	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, uIntercept))
13816	{
13817	VMXVEXITINFO ExitInfo;
13818	RT_ZERO(ExitInfo);
13819	ExitInfo.uReason = pVmxTransient->uExitReason;
13820	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13821	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13822	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
13823	}
13824	else
13825	{
13826	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
13827	rcStrict = vmxHCExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
13828	}
13829	}
13830	else
13831	{
13832	AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
13833	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
13834	}
13835	break;
13836	}
13837
13838	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
13839	{
13840	PCVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
13841	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
13842	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
13843	if ( (uGstHostMask & X86_CR0_TS)
13844	&& (uReadShadow & X86_CR0_TS))
13845	{
13846	VMXVEXITINFO ExitInfo;
13847	RT_ZERO(ExitInfo);
13848	ExitInfo.uReason = pVmxTransient->uExitReason;
13849	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13850	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13851	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
13852	}
13853	else
13854	rcStrict = vmxHCExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr);
13855	break;
13856	}
13857
13858	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
13859	{
13860	RTGCPTR GCPtrEffDst;
13861	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
13862	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
13863	if (fMemOperand)
13864	{
13865	vmxHCReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
13866	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
13867	}
13868	else
13869	GCPtrEffDst = NIL_RTGCPTR;
13870
13871	if (CPUMIsGuestVmxLmswInterceptSet(&pVCpu->cpum.GstCtx, uNewMsw))
13872	{
13873	VMXVEXITINFO ExitInfo;
13874	RT_ZERO(ExitInfo);
13875	ExitInfo.uReason = pVmxTransient->uExitReason;
13876	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13877	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
13878	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13879	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
13880	}
13881	else
13882	rcStrict = vmxHCExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, uNewMsw, GCPtrEffDst);
13883	break;
13884	}
13885
13886	default:
13887	{
13888	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
13889	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
13890	}
13891	}
13892
13893	if (rcStrict == VINF_IEM_RAISED_XCPT)
13894	{
13895	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13896	rcStrict = VINF_SUCCESS;
13897	}
13898	return rcStrict;
13899	}
13900
13901
13902	/**
13903	* Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
13904	* Conditional VM-exit.
13905	*/
13906	HMVMX_EXIT_DECL vmxHCExitMovDRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13907	{
13908	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13909
13910	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
13911	{
13912	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13913	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13914
13915	VMXVEXITINFO ExitInfo;
13916	RT_ZERO(ExitInfo);
13917	ExitInfo.uReason = pVmxTransient->uExitReason;
13918	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13919	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13920	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
13921	}
13922	return vmxHCExitMovDRx(pVCpu, pVmxTransient);
13923	}
13924
13925
13926	/**
13927	* Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
13928	* Conditional VM-exit.
13929	*/
13930	HMVMX_EXIT_DECL vmxHCExitIoInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13931	{
13932	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13933
13934	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
13935
13936	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
13937	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
13938	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
13939
13940	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
13941	uint8_t const cbAccess = s_aIOSizes[uIOSize];
13942	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
13943	{
13944	/*
13945	* IN/OUT instruction:
13946	* - Provides VM-exit instruction length.
13947	*
13948	* INS/OUTS instruction:
13949	* - Provides VM-exit instruction length.
13950	* - Provides Guest-linear address.
13951	* - Optionally provides VM-exit instruction info (depends on CPU feature).
13952	*/
13953	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
13954	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
13955
13956	/* Make sure we don't use stale/uninitialized VMX-transient info. below. */
13957	pVmxTransient->ExitInstrInfo.u = 0;
13958	pVmxTransient->uGuestLinearAddr = 0;
13959
13960	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
13961	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
13962	if (fIOString)
13963	{
13964	vmxHCReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
13965	if (fVmxInsOutsInfo)
13966	{
13967	Assert(RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
13968	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
13969	}
13970	}
13971
13972	VMXVEXITINFO ExitInfo;
13973	RT_ZERO(ExitInfo);
13974	ExitInfo.uReason = pVmxTransient->uExitReason;
13975	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
13976	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13977	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
13978	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
13979	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
13980	}
13981	return vmxHCExitIoInstr(pVCpu, pVmxTransient);
13982	}
13983
13984
13985	/**
13986	* Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
13987	*/
13988	HMVMX_EXIT_DECL vmxHCExitRdmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13989	{
13990	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13991
13992	uint32_t fMsrpm;
13993	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
13994	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap, pVCpu->cpum.GstCtx.ecx);
13995	else
13996	fMsrpm = VMXMSRPM_EXIT_RD;
13997
13998	if (fMsrpm & VMXMSRPM_EXIT_RD)
13999	{
14000	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
14001	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
14002	}
14003	return vmxHCExitRdmsr(pVCpu, pVmxTransient);
14004	}
14005
14006
14007	/**
14008	* Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
14009	*/
14010	HMVMX_EXIT_DECL vmxHCExitWrmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14011	{
14012	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14013
14014	uint32_t fMsrpm;
14015	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
14016	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap, pVCpu->cpum.GstCtx.ecx);
14017	else
14018	fMsrpm = VMXMSRPM_EXIT_WR;
14019
14020	if (fMsrpm & VMXMSRPM_EXIT_WR)
14021	{
14022	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
14023	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
14024	}
14025	return vmxHCExitWrmsr(pVCpu, pVmxTransient);
14026	}
14027
14028
14029	/**
14030	* Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
14031	*/
14032	HMVMX_EXIT_DECL vmxHCExitMwaitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14033	{
14034	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14035
14036	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
14037	{
14038	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
14039	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
14040	}
14041	return vmxHCExitMwait(pVCpu, pVmxTransient);
14042	}
14043
14044
14045	/**
14046	* Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
14047	* VM-exit.
14048	*/
14049	HMVMX_EXIT_DECL vmxHCExitMtfNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14050	{
14051	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14052
14053	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
14054	vmxHCReadGuestPendingDbgXctps(pVCpu, pVmxTransient);
14055	VMXVEXITINFO ExitInfo;
14056	RT_ZERO(ExitInfo);
14057	ExitInfo.uReason = pVmxTransient->uExitReason;
14058	ExitInfo.u64GuestPendingDbgXcpts = pVmxTransient->uGuestPendingDbgXcpts;
14059	return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
14060	}
14061
14062
14063	/**
14064	* Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
14065	*/
14066	HMVMX_EXIT_DECL vmxHCExitMonitorNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14067	{
14068	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14069
14070	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
14071	{
14072	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
14073	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
14074	}
14075	return vmxHCExitMonitor(pVCpu, pVmxTransient);
14076	}
14077
14078
14079	/**
14080	* Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
14081	*/
14082	HMVMX_EXIT_DECL vmxHCExitPauseNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14083	{
14084	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14085
14086	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
14087	* PAUSE when executing a nested-guest? If it does not, we would not need
14088	* to check for the intercepts here. Just call VM-exit... */
14089
14090	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
14091	if ( CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
14092	\|\| CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
14093	{
14094	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
14095	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
14096	}
14097	return vmxHCExitPause(pVCpu, pVmxTransient);
14098	}
14099
14100
14101	/**
14102	* Nested-guest VM-exit handler for when the TPR value is lowered below the
14103	* specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
14104	*/
14105	HMVMX_EXIT_NSRC_DECL vmxHCExitTprBelowThresholdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14106	{
14107	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14108
14109	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
14110	{
14111	vmxHCReadGuestPendingDbgXctps(pVCpu, pVmxTransient);
14112	VMXVEXITINFO ExitInfo;
14113	RT_ZERO(ExitInfo);
14114	ExitInfo.uReason = pVmxTransient->uExitReason;
14115	ExitInfo.u64GuestPendingDbgXcpts = pVmxTransient->uGuestPendingDbgXcpts;
14116	return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
14117	}
14118	return vmxHCExitTprBelowThreshold(pVCpu, pVmxTransient);
14119	}
14120
14121
14122	/**
14123	* Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
14124	* VM-exit.
14125	*/
14126	HMVMX_EXIT_DECL vmxHCExitApicAccessNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14127	{
14128	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14129
14130	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
14131	vmxHCReadIdtVectoringInfoVmcs(pVCpu, pVmxTransient);
14132	vmxHCReadIdtVectoringErrorCodeVmcs(pVCpu, pVmxTransient);
14133	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
14134
14135	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
14136
14137	Log4Func(("at offset %#x type=%u\n", VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual),
14138	VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual)));
14139
14140	VMXVEXITINFO ExitInfo;
14141	RT_ZERO(ExitInfo);
14142	ExitInfo.uReason = pVmxTransient->uExitReason;
14143	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
14144	ExitInfo.u64Qual = pVmxTransient->uExitQual;
14145
14146	VMXVEXITEVENTINFO ExitEventInfo;
14147	RT_ZERO(ExitEventInfo);
14148	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
14149	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
14150	return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
14151	}
14152
14153
14154	/**
14155	* Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
14156	* Conditional VM-exit.
14157	*/
14158	HMVMX_EXIT_DECL vmxHCExitApicWriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14159	{
14160	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14161
14162	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
14163	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
14164	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
14165	}
14166
14167
14168	/**
14169	* Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
14170	* Conditional VM-exit.
14171	*/
14172	HMVMX_EXIT_DECL vmxHCExitVirtEoiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14173	{
14174	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14175
14176	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
14177	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
14178	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
14179	}
14180
14181
14182	/**
14183	* Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
14184	*/
14185	HMVMX_EXIT_DECL vmxHCExitRdtscpNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14186	{
14187	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14188
14189	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
14190	{
14191	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
14192	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
14193	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
14194	}
14195	return vmxHCExitRdtscp(pVCpu, pVmxTransient);
14196	}
14197
14198
14199	/**
14200	* Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
14201	*/
14202	HMVMX_EXIT_NSRC_DECL vmxHCExitWbinvdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14203	{
14204	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14205
14206	if (CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
14207	{
14208	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
14209	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
14210	}
14211	return vmxHCExitWbinvd(pVCpu, pVmxTransient);
14212	}
14213
14214
14215	/**
14216	* Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
14217	*/
14218	HMVMX_EXIT_DECL vmxHCExitInvpcidNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14219	{
14220	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14221
14222	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
14223	{
14224	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
14225	vmxHCReadExitInstrLenVmcs(ppVCpu, VmxTransient);
14226	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
14227	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
14228
14229	VMXVEXITINFO ExitInfo;
14230	RT_ZERO(ExitInfo);
14231	ExitInfo.uReason = pVmxTransient->uExitReason;
14232	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
14233	ExitInfo.u64Qual = pVmxTransient->uExitQual;
14234	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
14235	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
14236	}
14237	return vmxHCExitInvpcid(pVCpu, pVmxTransient);
14238	}
14239
14240
14241	/**
14242	* Nested-guest VM-exit handler for invalid-guest state
14243	* (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
14244	*/
14245	HMVMX_EXIT_DECL vmxHCExitErrInvalidGuestStateNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14246	{
14247	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14248
14249	/*
14250	* Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
14251	* So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
14252	* Handle it like it's in an invalid guest state of the outer guest.
14253	*
14254	* When the fast path is implemented, this should be changed to cause the corresponding
14255	* nested-guest VM-exit.
14256	*/
14257	return vmxHCExitErrInvalidGuestState(pVCpu, pVmxTransient);
14258	}
14259
14260
14261	/**
14262	* Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
14263	* and only provide the instruction length.
14264	*
14265	* Unconditional VM-exit.
14266	*/
14267	HMVMX_EXIT_DECL vmxHCExitInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14268	{
14269	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14270
14271	#ifdef VBOX_STRICT
14272	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14273	switch (pVmxTransient->uExitReason)
14274	{
14275	case VMX_EXIT_ENCLS:
14276	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
14277	break;
14278
14279	case VMX_EXIT_VMFUNC:
14280	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_VMFUNC));
14281	break;
14282	}
14283	#endif
14284
14285	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
14286	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
14287	}
14288
14289
14290	/**
14291	* Nested-guest VM-exit handler for instructions that provide instruction length as
14292	* well as more information.
14293	*
14294	* Unconditional VM-exit.
14295	*/
14296	HMVMX_EXIT_DECL vmxHCExitInstrWithInfoNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14297	{
14298	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14299
14300	#ifdef VBOX_STRICT
14301	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14302	switch (pVmxTransient->uExitReason)
14303	{
14304	case VMX_EXIT_GDTR_IDTR_ACCESS:
14305	case VMX_EXIT_LDTR_TR_ACCESS:
14306	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
14307	break;
14308
14309	case VMX_EXIT_RDRAND:
14310	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
14311	break;
14312
14313	case VMX_EXIT_RDSEED:
14314	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
14315	break;
14316
14317	case VMX_EXIT_XSAVES:
14318	case VMX_EXIT_XRSTORS:
14319	/** @todo NSTVMX: Verify XSS-bitmap. */
14320	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
14321	break;
14322
14323	case VMX_EXIT_UMWAIT:
14324	case VMX_EXIT_TPAUSE:
14325	Assert(CPUMIsGuestVmxProcCtlsSet(pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
14326	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
14327	break;
14328
14329	case VMX_EXIT_LOADIWKEY:
14330	Assert(CPUMIsGuestVmxProcCtls3Set(pCtx, VMX_PROC_CTLS3_LOADIWKEY_EXIT));
14331	break;
14332	}
14333	#endif
14334
14335	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
14336	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
14337	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
14338
14339	VMXVEXITINFO ExitInfo;
14340	RT_ZERO(ExitInfo);
14341	ExitInfo.uReason = pVmxTransient->uExitReason;
14342	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
14343	ExitInfo.u64Qual = pVmxTransient->uExitQual;
14344	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
14345	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
14346	}
14347
14348	/** @} */
14349	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
14350

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/VMXAllTemplate.cpp.h@ 93115

Download in other formats: